def stream(self, fileobj, callback=None, sleeptime=5):
"""
Stream the bundle to the fileobj.
This method is a blocking I/O operation.
The ``fileobj`` should be an open file like object with 'wb' options.
An asynchronous callback method MAY be provided via the optional ``callback``
keyword argument. Periodically, the callback method is provided with the current
percentage of completion.
"""
notifythread = None
if callable(callback):
self._save_total_size()
notifythread = self._setup_notify_thread(callback, sleeptime)
tarfile = taropen(None, 'w|', fileobj)
for file_data in self.file_data:
tarinfo, fileobj = self._tarinfo_from_file_data(file_data)
tarfile.addfile(tarinfo, fileobj)
self.md_obj.append(
self._build_file_info(file_data, fileobj.hashdigest()))
md_txt = bytes(metadata_encode(self.md_obj), 'utf8')
md_fd = StringIO(md_txt)
md_tinfo = TarInfo('metadata.txt')
md_tinfo.size = len(md_txt)
tarfile.addfile(md_tinfo, md_fd)
tarfile.close()
self._complete = True
if callable(callback):
notifythread.join()
def copy_to_container(container: "Container", source_path: str,
target_path: str) -> None:
"""
Copy a file into a Docker container
:param container: Container object
:param source_path: Source file path
:param target_path: Target file path (in the container)
:return:
"""
# https://github.com/docker/docker-py/issues/1771
with open(source_path, "rb") as f:
data = f.read()
tarinfo = TarInfo(name=os.path.basename(target_path))
tarinfo.size = len(data)
tarinfo.mtime = int(time.time())
stream = BytesIO()
tar = TarFile(fileobj=stream, mode="w")
tar.addfile(tarinfo, BytesIO(data))
tar.close()
stream.seek(0)
container.put_archive(path=os.path.dirname(target_path),
data=stream.read())
def _dump_files(self, tar):
"""
Dump all uploaded media to the archive.
"""
# Loop through all models and find FileFields
for model in apps.get_models():
# Get the name of all file fields in the model
field_names = []
for field in model._meta.fields:
if isinstance(field, models.FileField):
field_names.append(field.name)
# If any were found, loop through each row
if len(field_names):
for row in model.objects.all():
for field_name in field_names:
field = getattr(row, field_name)
if field:
field.open()
info = TarInfo(field.name)
info.size = field.size
tar.addfile(info, field)
field.close()
def add_buf_to_tar(tar: TarFile, filename: str, buf: BytesIO):
buf.flush()
buf.seek(0)
info = TarInfo(name=filename)
info.size = len(buf.getvalue())
tar.addfile(info, buf)
def sanitize_tarinfo(tarinfo: tarfile.TarInfo):
path = Path(tarinfo.name)
# Ban absolute paths
if path.is_absolute():
return False
# Ban paths containing .. that would go outside
try:
base = Path("fake_path")
base.joinpath(path).relative_to(base)
except ValueError as e:
logger.warning(f"Unsafe path {path}")
return False
# Ban links that would point somewhere outside
if tarinfo.islnk() or tarinfo.issym():
try:
base = Path("fake_path")
link = path.parent / tarinfo.linkname
base.joinpath(link).relative_to(base)
except ValueError as e:
logger.warning(f"Unsafe path {path}")
return False
return True
def test_09__verify_version(self, mock_isfile, mock_fumktmp, mock_osbase,
mock_fugetdata, mock_furm, mock_versionok):
"""Test09 UdockerTools()._verify_version()."""
tball = "/home/udocker.tar"
mock_isfile.return_value = False
utools = UdockerTools(self.local)
status = utools._verify_version(tball)
self.assertTrue(mock_isfile.called)
self.assertEqual(status, (False, ""))
tball = "/home/udocker.tar"
mock_isfile.return_value = True
mock_fumktmp.return_value = ""
utools = UdockerTools(self.local)
status = utools._verify_version(tball)
self.assertTrue(mock_isfile.called)
self.assertTrue(mock_fumktmp.called)
self.assertEqual(status, (False, ""))
tball = "/home/udocker.tar"
tinfo1 = TarInfo("udocker_dir/lib/VERSION")
tinfo2 = TarInfo("a")
mock_isfile.return_value = True
mock_fumktmp.return_value = "/home/tmp"
mock_osbase.return_value = "VERSION"
mock_fugetdata.return_value = "1.2.7"
mock_furm.return_value = None
mock_versionok.return_value = True
with patch.object(tarfile, 'open', autospec=True) as open_mock:
open_mock.return_value.getmembers.return_value = [tinfo2, tinfo1]
open_mock.return_value.extract.return_value = None
utools = UdockerTools(self.local)
status = utools._verify_version(tball)
self.assertEqual(status, (True, "1.2.7"))
self.assertTrue(mock_furm.called)
def add(self, filename, size, fileobj):
"""
Add the provided file to the archive
"""
tarinfo = TarInfo(filename)
tarinfo.size = size
self._tarfile.addfile(tarinfo, fileobj)
def upload(self):
logger.info("Loading artifacts")
t = ts()
tar_data = io.BytesIO()
with tarfile.open(fileobj=tar_data, mode="w|") as tar:
for root, _, files in os.walk(self._artifact_directory):
for af in files:
full_path = os.path.join(root, af)
relpath = os.path.relpath(full_path, self._artifact_directory)
ti = TarInfo(relpath)
stat = os.stat(full_path)
ti.size = stat.st_size
ti.mode = stat.st_mode
with open(full_path, "rb") as f:
tar.addfile(ti, f)
res = self._container.put_archive(config.build_dir, tar_data.getvalue())
if not res:
raise Exception(f"Error loading artifact: {af}")
t = ts() - t
logger.info("Artifacts loaded in %.3fs", t)
def stream_regular_file(self, filepath, tarinfo_buf, file_info):
try:
file_system = files.get_fs_by_file_path(filepath)
with file_system.open(filepath, 'rb') as fd:
f_iter = iter(lambda: fd.read(self.CHUNKSIZE), '') # pylint: disable=cell-var-from-loop
try:
yield tarinfo_buf
chunk = ''
for chunk in f_iter:
yield chunk
if len(chunk) % self.BLOCKSIZE != 0:
yield (self.BLOCKSIZE -
(len(chunk) % self.BLOCKSIZE)) * b'\0'
except (IOError, fs.errors.OperationFailed):
msg = (
"Error happened during sending file content in archive stream, file path: %s, "
"container: %s/%s, archive path: %s" % file_info)
self.log.critical(msg)
self.abort(500, msg)
except (fs.errors.ResourceNotFound, fs.errors.OperationFailed,
IOError):
self.log.critical(
"Couldn't find the file during creating archive stream: %s, "
"container: %s/%s, archive path: %s" % file_info)
tarinfo = TarInfo()
tarinfo.name = file_info[3] + '.MISSING'
yield tarinfo.tobuf()
def _test_roundtrip(self, context):
path = context.given_file()
content = b'test content'
filename = 'archived-file.txt'
with xtarfile_open(path, context.mode('w')) as archive:
buffer = BytesIO()
buffer.write(content)
buffer.seek(0)
tarinfo = TarInfo()
tarinfo.size = len(content)
tarinfo.name = filename
archive.addfile(tarinfo, buffer)
with xtarfile_open(path, context.mode('r')) as archive:
while True:
member = archive.next()
if member is None:
self.fail('{} not found in archive'.format(filename))
if member.name == filename:
buffer = archive.extractfile(member)
actual_content = buffer.read()
break
self.assertEqual(actual_content, content)
def _addToArchive(output, filename, archive):
output.seek(0)
xmlInfo = TarInfo(filename)
xmlInfo.size = len(output.getbuffer())
xmlInfo.mtime = time.time()
archive.addfile(xmlInfo,output)
output.close()
def put(self, content, filename="", file_hash=""):
""" Store file information in hashed tree """
if not filename and not file_hash:
raise ValueError('Filename or FileHash is mandatory')
if filename:
# File accesibility
if not os.path.exists(filename):
raise IOError('Unaccesible file %s', filename)
# Calc hash
file_hash = self._get_hash(filename)
if not file_hash:
raise ValueError('Hash of file is mandatory')
# Get file path for hash
path, tarfile, hashed_filename = self._get_path(file_hash)
# Create file path
try:
os.makedirs(path)
except WindowsError:
pass
except OSError:
pass
# Open tarfile
if self.external_compressor:
# External compressor is not suited for adding files.
raise ValueError('You cannot use external compressor for write files')
with TarFile.open(name=os.path.join(path, tarfile), mode='a') as tar:
with FileLock(os.path.join(path, tarfile)) as lock:
# Test if file already exists into tarfile
try:
tar.getmember(hashed_filename)
raise ValueError('Member already exists')
except KeyError:
pass
except:
raise
data = self.encoder.encode(content)
if self.internal_compressor:
data = self.internal_compressor.compress(data)
data_file = StringIO(data)
mtime = time.time()
ti = TarInfo(hashed_filename)
ti.size = data_file.len
ti.mtime = mtime
tar.addfile(tarinfo=ti, fileobj=data_file)
tar.close()
return file_hash
def writeDataFile(self, filename, text, content_type, subdir=None):
""" See IExportContext.
"""
if subdir is not None:
filename = '/'.join((subdir, filename))
parents = filename.split('/')[:-1]
while parents:
path = '/'.join(parents) + '/'
if path not in self._archive.getnames():
info = TarInfo(path)
info.type = DIRTYPE
# tarfile.filemode(0755) == '-rwxr-xr-x'
info.mode = 0755
info.mtime = time.time()
self._archive.addfile(info)
parents.pop()
info = TarInfo(filename)
if isinstance(text, str):
stream = StringIO(text)
info.size = len(text)
elif isinstance(text, unicode):
raise ValueError("Unicode text is not supported, even if it only "
"contains ascii. Please encode your data")
else:
# Assume text is a an instance of a class like
# Products.Archetypes.WebDAVSupport.PdataStreamIterator,
# as in the case of ATFile
stream = text.file
info.size = text.size
info.mtime = time.time()
self._archive.addfile(info, stream)
def writeDataFile(self, filename, text, content_type, subdir=None):
""" See IExportContext.
"""
if subdir is not None:
filename = '/'.join((subdir, filename))
parents = filename.split('/')[:-1]
while parents:
path = '/'.join(parents) + '/'
if path not in self._archive.getnames():
info = TarInfo(path)
info.type = DIRTYPE
# tarfile.filemode(0o755) == '-rwxr-xr-x'
info.mode = 0o755
info.mtime = time.time()
self._archive.addfile(info)
parents.pop()
info = TarInfo(filename)
if isinstance(text, six.text_type):
encoding = self.getEncoding() or 'utf-8'
text = text.encode(encoding)
if isinstance(text, six.binary_type):
stream = BytesIO(text)
info.size = len(text)
else:
# Assume text is a an instance of a class like
# Products.Archetypes.WebDAVSupport.PdataStreamIterator,
# as in the case of ATFile
stream = text.file
info.size = text.size
info.mtime = time.time()
self._archive.addfile(info, stream)
def test_10__install(self, mock_isfile, mock_futil, mock_osbase):
"""Test10 UdockerTools()._install()."""
tfile = ""
mock_isfile.return_value = False
utools = UdockerTools(self.local)
status = utools._install(tfile)
self.assertFalse(status)
tinfo1 = TarInfo("udocker_dir/bin/ls")
tinfo2 = TarInfo("udocker_dir/lib/lib1")
tfile = "udocker.tar"
mock_isfile.return_value = True
mock_futil.return_value.chmod.return_value = None
mock_futil.return_value.rchmod.side_effect = [
None, None, None, None, None, None
]
mock_osbase.side_effect = ["ls", "ls", "lib1", "lib1", "doc", "doc1"]
self.local.create_repo.return_value = None
with patch.object(tarfile, 'open', autospec=True) as open_mock:
open_mock.return_value.getmembers.side_effect = [[tinfo1, tinfo2],
[tinfo1, tinfo2],
[tinfo1, tinfo2]]
open_mock.return_value.extract.side_effect = [None, None]
utools = UdockerTools(self.local)
status = utools._install(tfile)
self.assertTrue(status)
self.assertTrue(mock_futil.called)
self.assertTrue(mock_futil.return_value.rchmod.call_count, 4)
def GetTarInfo(filename, filetype=DIRTYPE, mode=0755):
"""Create information for tar files"""
tarinfo = TarInfo(path.basename(filename))
tarinfo.type = filetype
tarinfo.mode = mode
tarinfo.mtime = time()
return tarinfo
def stream(self, fileobj, callback=None, sleeptime=5):
"""
Stream the bundle to the fileobj.
The fileobj should be an open file like object with 'wb' options.
If the callback is given then percent complete of the size
of the bundle will be given to the callback as the first argument.
"""
notifythread = None
if callable(callback):
self._save_total_size()
notifythread = self._setup_notify_thread(callback, sleeptime)
tarfile = taropen(None, 'w|', fileobj)
for file_data in self.file_data:
tarinfo, fileobj = self._tarinfo_from_file_data(file_data)
tarfile.addfile(tarinfo, fileobj)
self.md_obj.append(
self._build_file_info(file_data, fileobj.hashdigest()))
md_txt = metadata_encode(self.md_obj)
md_txt = md_txt if PY2 else bytes(md_txt, 'UTF-8')
md_fd = StringIO(md_txt)
md_tinfo = TarInfo('metadata.txt')
md_tinfo.size = len(md_txt)
tarfile.addfile(md_tinfo, md_fd)
tarfile.close()
self._complete = True
if callable(callback):
notifythread.join()
def test_basic(self):
# writing
db = TarDB(dirname).open(mode='w')
data_foo = '123'
data_bar = 'ABCDEF'
db.add_record(TarInfo('foo'), data_foo)
db.add_record(TarInfo('bar'), data_bar)
db.close()
#
files = os.listdir(dirname)
self.assertEqual(len(files), 3)
self.assertTrue('catalog' in files)
self.assertTrue('lock' in files)
self.assertTrue('db00000.tar' in files)
# reading
db = TarDB(dirname).open()
(info1, data1) = db.get_record(0)
self.assertEqual(data1, data_foo)
self.assertEqual(len(data1), info1.size)
(info2, data2) = db.get_record(1)
self.assertEqual(data2, data_bar)
self.assertEqual(len(data2), info2.size)
# iter
infos = list(db)
self.assertEqual(len(infos), 2)
self.assertEqual(infos[0].name, info1.name)
self.assertEqual(infos[1].name, info2.name)
db.close()
return
def test_rawtar(self):
"""Create a normal tar archive and restore it"""
raw = BytesIO()
tarfile = TarFile(mode='w', fileobj=raw)
testdata = rand_str(20) * 5000
inf = TarInfo("simpletar")
fileraw = BytesIO()
fileraw.write(testdata)
inf.size = len(testdata)
fileraw.seek(0)
tarfile.addfile(inf, fileobj=fileraw)
tarfile.close()
raw.seek(0)
data = raw.read()
cnt = rand_str(20)
ret = requests.put(self.make_uri("restore", container=cnt), data=data)
self.assertEqual(ret.status_code, 201)
meta, stream = self.conn.object_fetch(self.account, cnt, "simpletar")
self.assertEqual(
md5("".join(stream)).hexdigest(),
md5(testdata).hexdigest())
def move_certs(self, paths):
self.log.info("Staging internal ssl certs for %s", self._log_name)
yield self.pull_image(self.move_certs_image)
# create the volume
volume_name = self.format_volume_name(self.certs_volume_name, self)
# create volume passes even if it already exists
self.log.info("Creating ssl volume %s for %s", volume_name, self._log_name)
yield self.docker('create_volume', volume_name)
# create a tar archive of the internal cert files
# docker.put_archive takes a tarfile and a running container
# and unpacks the archive into the container
nb_paths = {}
tar_buf = BytesIO()
archive = TarFile(fileobj=tar_buf, mode='w')
for key, hub_path in paths.items():
fname = os.path.basename(hub_path)
nb_paths[key] = '/certs/' + fname
with open(hub_path, 'rb') as f:
content = f.read()
tarinfo = TarInfo(name=fname)
tarinfo.size = len(content)
tarinfo.mtime = os.stat(hub_path).st_mtime
tarinfo.mode = 0o644
archive.addfile(tarinfo, BytesIO(content))
archive.close()
tar_buf.seek(0)
# run a container to stage the certs,
# mounting the volume at /certs/
host_config = self.client.create_host_config(
binds={
volume_name: {"bind": "/certs", "mode": "rw"},
},
)
container = yield self.docker('create_container',
self.move_certs_image,
volumes=["/certs"],
host_config=host_config,
)
container_id = container['Id']
self.log.debug(
"Container %s is creating ssl certs for %s",
container_id[:12], self._log_name,
)
# start the container
yield self.docker('start', container_id)
# stage the archive to the container
try:
yield self.docker(
'put_archive',
container=container_id,
path='/certs',
data=tar_buf,
)
finally:
yield self.docker('remove_container', container_id)
return nb_paths
def move_certs(self, paths):
self.log.info("Staging internal ssl certs for %s", self._log_name)
yield self.pull_image(self.move_certs_image)
# create the volume
volume_name = self.format_volume_name(self.certs_volume_name, self)
# create volume passes even if it already exists
self.log.info("Creating ssl volume %s for %s", volume_name, self._log_name)
yield self.docker('create_volume', volume_name)
# create a tar archive of the internal cert files
# docker.put_archive takes a tarfile and a running container
# and unpacks the archive into the container
nb_paths = {}
tar_buf = BytesIO()
archive = TarFile(fileobj=tar_buf, mode='w')
for key, hub_path in paths.items():
fname = os.path.basename(hub_path)
nb_paths[key] = '/certs/' + fname
with open(hub_path, 'rb') as f:
content = f.read()
tarinfo = TarInfo(name=fname)
tarinfo.size = len(content)
tarinfo.mtime = os.stat(hub_path).st_mtime
tarinfo.mode = 0o644
archive.addfile(tarinfo, BytesIO(content))
archive.close()
tar_buf.seek(0)
# run a container to stage the certs,
# mounting the volume at /certs/
host_config = self.client.create_host_config(
binds={
volume_name: {"bind": "/certs", "mode": "rw"},
},
)
container = yield self.docker('create_container',
self.move_certs_image,
volumes=["/certs"],
host_config=host_config,
)
container_id = container['Id']
self.log.debug(
"Container %s is creating ssl certs for %s",
container_id[:12], self._log_name,
)
# start the container
yield self.docker('start', container_id)
# stage the archive to the container
try:
yield self.docker(
'put_archive',
container=container_id,
path='/certs',
data=tar_buf,
)
finally:
yield self.docker('remove_container', container_id)
return nb_paths
def generate_tar(entries):
tar_buf = BytesIO()
tar_file = TarFile(mode="w", fileobj=tar_buf)
for path, contents in entries.items():
tar_info = TarInfo(name=path)
tar_info.size = len(contents)
tar_file.addfile(tar_info, fileobj=BytesIO(contents))
return BytesIO(tar_buf.getvalue())
def generate_tar(entries):
tar_buf = BytesIO()
tar_file = TarFile(mode='w', fileobj=tar_buf)
for path, contents in entries.items():
tar_info = TarInfo(name=path)
tar_info.size = len(contents)
tar_file.addfile(tar_info, fileobj=BytesIO(contents))
return BytesIO(tar_buf.getvalue())
def file_filter(info: tarfile.TarInfo):
info.mode = 0o00777 if executable else 0o00666
info.mtime = 0
info.type = tarfile.REGTYPE
info.uid = info.gid = 0
info.uname = info.gname = "root"
info.pax_headers = {}
return info
def addFile(tar, dest, file, file_size):
if dest not in written_files:
info = TarInfo(dest)
info.size = file_size
info.mtime = now
info.mode = 0777
tar.addfile(info, fileobj=file)
written_files.add(dest)
def addFile(tar, dest, file, file_size):
if dest not in written_files:
info = TarInfo(dest)
info.size = file_size
info.mtime = now
info.mode = 0o777
tar.addfile(info, fileobj=file)
written_files.add(dest)
def compactar(idfiltro):
m_filtro = Filtro.objects.get(pk=idfiltro)
if m_filtro.situacao in SITUACOES_EXECUTORES:
return
m_filtro.situacao = "6"
m_filtro.percentual_atual = 0
m_filtro.save()
slug_classificador = slugify(m_filtro.nome)
documentos = m_filtro.documento_set.all()
qtd_documentos = documentos.count()
# cria o streamfile em disco
nometar = "%s.tar.bz2" % slug_classificador
numeros_documentos = defaultdict(int)
with BytesIO() as arquivotar:
tarfile = TarFile(name=nometar, mode="w", fileobj=arquivotar)
for contador, documento in enumerate(documentos):
numero = documento.numero
numeros_documentos[numero] += 1
ordem = numeros_documentos[numero]
with BytesIO() as conteudo_documento:
conteudo_documento.write(
documento.conteudo.encode("latin1", "ignore"))
conteudo_documento.seek(0)
if documento.classe_filtro:
classe = slugify(documento.classe_filtro.nome)
else:
classe = slugify("Não Identificado")
if documento.tipo_movimento:
tipo = slugify(documento.tipo_movimento.nome)
else:
tipo = "documento"
tarinfo = TarInfo(name="%s/%s-%s-%s.txt" %
(classe, tipo, numero, ordem))
tarinfo.size = len(conteudo_documento.getvalue())
conteudo_documento.seek(0)
tarfile.addfile(fileobj=conteudo_documento, tarinfo=tarinfo)
m_filtro.percentual_atual = contador / qtd_documentos * 100
logger.info("Percentual %s" % m_filtro.percentual_atual)
m_filtro.save()
arquivotar.seek(0)
m_filtro.saida.save(nometar, File(arquivotar))
m_filtro.situacao = "7"
m_filtro.save()
def get_tarinfo(name, buffer: typing.Union[BytesIO, StringIO]):
tar_info = TarInfo(name=name)
buffer.seek(0)
if isinstance(buffer, BytesIO):
tar_info.size = len(buffer.getbuffer())
else:
tar_info.size = len(buffer.getvalue())
tar_info.mtime = datetime.now().timestamp()
return tar_info
def _dump_meta(self, tar):
"""
Dump metadata to the archive.
"""
data = MixedIO()
dump({'version': __version__}, data)
info = TarInfo('meta.json')
info.size = data.rewind()
tar.addfile(info, data)
def _given_resource(self, resource_id: str, name: str, lines: bytes):
client = self._storage._file_storage._client
mode = f'w:{Path(resource_id).suffix[1:]}'
with removing(create_tempfilename(resource_id)) as buffer_path:
with tarfile_open(buffer_path, mode) as archive:
tarinfo = TarInfo(name)
tarinfo.size = len(lines)
archive.addfile(tarinfo, BytesIO(lines))
client.upload_object(buffer_path, resource_id)
def add_bytes_to_tar(tar: TarFile, filename: str, data: bytes):
buf = BytesIO()
buf.write(data)
buf.flush()
buf.seek(0)
info = TarInfo(name=filename)
info.size = len(data)
tar.addfile(info, buf)
def _given_resource(self, resource_id: str, name: str, lines: bytes):
client = self._storage._file_storage._client
buffer = BytesIO()
with tarfile_open(mode='w:gz', fileobj=buffer) as archive:
tarinfo = TarInfo(name)
tarinfo.size = len(lines)
archive.addfile(tarinfo, BytesIO(lines))
buffer.seek(0)
client.upload_object_via_stream(buffer, resource_id)
def create_dir(self, path):
"""Create a directory within the tarfile.
:param path: the path to put the directory at.
"""
tarinfo = TarInfo(name=path)
tarinfo.type = DIRTYPE
tarinfo.mode = 0755
self._set_defaults(tarinfo)
self.addfile(tarinfo)
def create_dir(self, path):
"""Create a directory within the tarfile.
:param path: the path to put the directory at.
"""
tarinfo = TarInfo(name=path)
tarinfo.type = DIRTYPE
tarinfo.mode = 0755
self._set_defaults(tarinfo)
self.addfile(tarinfo)
def writeDataFile( self, filename, text, content_type, subdir=None ):
""" See IExportContext.
"""
mod_time = time.time()
if subdir is not None:
elements = subdir.split('/')
parents = filter(None, elements)
while parents:
dirname = os.path.join(*parents)
try:
self._archive.getmember(dirname+'/')
except KeyError:
info = TarInfo(dirname)
info.size = 0
info.mode = 509
info.mtime = mod_time
info.type = DIRTYPE
self._archive.addfile(info, StringIO())
parents = parents[:-1]
filename = '/'.join( ( subdir, filename ) )
stream = StringIO( text )
info = TarInfo( filename )
info.size = len( text )
info.mode = 436
info.mtime = mod_time
self._archive.addfile( info, stream )
def write(self, file_name):
if not self.data or not os.path.isdir(self.data):
raise Exception('Must set data before building')
gzfile = GzipFile(file_name, 'w')
tar = TarFile(fileobj=gzfile, mode='w')
buff = BytesIO(json.dumps(self.control).encode())
info = TarInfo(name='./CONTROL')
info.size = buff.getbuffer().nbytes
tar.addfile(tarinfo=info, fileobj=buff)
if self.init is not None:
buff = BytesIO(self.init.encode())
info = TarInfo(name='./INIT')
info.size = buff.getbuffer().nbytes
tar.addfile(tarinfo=info, fileobj=buff)
data = BytesIO()
datatar = TarFile(fileobj=data, mode='w')
datatar.add(self.data, '/')
datatar.close()
data.seek(0)
info = TarInfo(name='./DATA')
info.size = data.getbuffer().nbytes
tar.addfile(tarinfo=info, fileobj=data)
tar.close()
gzfile.close()
def addString(tar, dest, string):
if dest not in written_files:
print dest, string
info = TarInfo(dest)
info.size = len(string)
info.mtime = now
info.mode = 0777
file = StringIO(string)
tar.addfile(info, fileobj=file)
file.close()
written_files.add(dest)
def addString(tar, dest, string):
if dest not in written_files:
print dest, string
info = TarInfo(dest)
info.size = len(string)
info.mtime = now
info.mode = 0777
file = StringIO(string)
tar.addfile(info, fileobj=file)
file.close()
written_files.add(dest)
def addBytes(tar, dest, bytes):
if dest not in written_files:
# print dest, string
info = TarInfo(dest)
info.size = len(bytes)
info.mtime = now
info.mode = 0o777
file = BytesIO(bytes)
tar.addfile(info, fileobj=file)
file.close()
written_files.add(dest)
def create_article(self, data, info=None):
if not self._mode: raise TarCMS.TarCMSError('not open: %r' % self)
if info is None:
info = TarInfo()
assert isinstance(info, TarInfo)
aid = '%08x' % self._artdb.nextrecno()
info.name = aid+info.name
tid = self._add_corpus(info, data)
assert aid == tid
self._artdb.add_record(tid)
return aid
def given_download(self, payload: Dict[str, bytes], compression: str):
resource_id = '{}.tar.{}'.format(uuid4(), compression)
download_filename = join(self._content_root, resource_id)
with self.sync._open(download_filename, 'w') as archive:
for filename, content in payload.items():
tarinfo = TarInfo(filename)
tarinfo.size = len(content)
archive.addfile(tarinfo, BytesIO(content))
self.email_server_client_mock.download.return_value = resource_id
def create_file_from_string(self, filename, content):
"""Create a file with the contents passed as a string.
:param filename: the path to put the file at inside the
tarfile.
:param content: the content to put in the created file.
"""
tarinfo = TarInfo(name=filename)
tarinfo.size = len(content)
self._set_defaults(tarinfo)
fileobj = StringIO(content)
self.addfile(tarinfo, fileobj=fileobj)
def writeDataFile( self, filename, text, content_type, subdir=None ):
""" See IExportContext.
"""
if subdir is not None:
filename = '/'.join( ( subdir, filename ) )
stream = StringIO( text )
info = TarInfo( filename )
info.size = len( text )
info.mtime = time.time()
self._archive.addfile( info, stream )
def _dump_db(self, tar):
"""
Dump the rows in each model to the archive.
"""
# Dump the tables to a MixedIO
data = MixedIO()
call_command('dumpdata', all=True, format='json', indent=self.attr.get('ARCHIVE_DB_INDENT'),
exclude=self.attr.get('ARCHIVE_EXCLUDE'), stdout=data)
info = TarInfo(DB_DUMP)
info.size = data.rewind()
tar.addfile(info, data)
def _addMember(filename, data, modtime):
from tarfile import DIRTYPE
parents = filename.split('/')[:-1]
while parents:
path = '/'.join(parents) + '/'
if path not in archive.getnames():
info = TarInfo(path)
info.type = DIRTYPE
info.mtime = modtime
archive.addfile(info)
parents.pop()
_addOneMember(filename, data, modtime)
def _addMember(path, data, modtime):
from tarfile import DIRTYPE
elements = path.split('/')
parents = filter(None, [elements[x] for x in range(len(elements))])
for parent in parents:
info = TarInfo()
info.name = parent
info.size = 0
info.mtime = mod_time
info.type = DIRTYPE
archive.addfile(info, StringIO())
_addOneMember(path, data, modtime)
def write_package():
tbs = ForgeClient.UPLOAD_TAR_BUFFER_SIZE
with TarFile.open(mode="w|gz", fileobj=body, bufsize=tbs,
dereference=True) as tar:
for file in files:
self.debug("Sending %s", file)
ti = TarInfo(file)
fp = os.path.join(self.path, file)
ti.size = os.path.getsize(fp)
ti.mode = 0o666
with open(fp, "rb") as fd:
tar.addfile(ti, fileobj=fd)
body.close()
def writeDataFile( self, filename, text, content_type, subdir=None ):
""" See IExportContext.
"""
if subdir is not None:
filename = '/'.join( ( subdir, filename ) )
parents = filename.split('/')[:-1]
while parents:
path = '/'.join(parents) + '/'
if path not in self._archive.getnames():
info = TarInfo(path)
info.type = DIRTYPE
# tarfile.filemode(0755) == '-rwxr-xr-x'
info.mode = 0755
info.mtime = time.time()
self._archive.addfile(info)
parents.pop()
info = TarInfo(filename)
if isinstance(text, basestring):
stream = StringIO(text)
info.size = len(text)
else:
# Assume text is a an instance of a class like
# Products.Archetypes.WebDAVSupport.PdataStreamIterator,
# as in the case of ATFile
stream = text.file
info.size = text.size
info.mtime = time.time()
self._archive.addfile( info, stream )
def writeDataFile( self, filename, text, content_type, subdir=None ):
""" See IExportContext.
"""
if subdir is not None:
filename = '/'.join( ( subdir, filename ) )
parents = filename.split('/')[:-1]
while parents:
path = '/'.join(parents) + '/'
if path not in self._archive.getnames():
info = TarInfo(path)
info.type = DIRTYPE
# tarfile.filemode(0755) == '-rwxr-xr-x'
info.mode = 0755
info.mtime = time.time()
self._archive.addfile(info)
parents.pop()
info = TarInfo(filename)
if isinstance(text, str):
stream = StringIO(text)
info.size = len(text)
elif isinstance(text, unicode):
raise ValueError("Unicode text is not supported, even if it only "
"contains ascii. Please encode your data. See "
"GS 1.7.0 changes for more")
else:
# Assume text is a an instance of a class like
# Products.Archetypes.WebDAVSupport.PdataStreamIterator,
# as in the case of ATFile
stream = text.file
info.size = text.size
info.mtime = time.time()
self._archive.addfile( info, stream )
def _dump_meta(self, tar):
"""
Dump metadata to the archive.
"""
data = MixedIO()
meta_dict = OrderedDict((
('version', __version__),
('db_file', DB_DUMP),
('media_folder', MEDIA_DIR),
('settings', self.attr.settings_dict()),
))
dump(meta_dict, data, indent=2)
info = TarInfo(META_DUMP)
info.size = data.rewind()
tar.addfile(info, data)
def test_change_info(self):
# writing
db = TarDB(dirname).open('r+')
data_foo = '123'
mtime = 12345
info = TarInfo('foo')
info.mtime = mtime
db.add_record(info, data_foo)
db.close()
# reading
db = TarDB(dirname).open('r+')
info = db[0]
self.assertEqual(info.mtime, mtime)
db[0] = info
db.close()
return
def run(self, args, argv):
# Create a temporary tarball with our whole build context and
# dockerfile for the update
tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz")
tmp_tar = TarFile(fileobj=tmp, mode='w')
# Add the executable to the tarball, using the current
# configured binfmt_misc path. If we don't get a path then we
# only need the support libraries copied
ff, enabled = _check_binfmt_misc(args.executable)
if not enabled:
print("binfmt_misc not enabled, update disabled")
return 1
if ff:
tmp_tar.add(args.executable, arcname=ff)
# Add any associated libraries
libs = _get_so_libs(args.executable)
if libs:
for l in libs:
tmp_tar.add(os.path.realpath(l), arcname=l)
# Create a Docker buildfile
df = StringIO()
df.write("FROM %s\n" % args.tag)
df.write("ADD . /\n")
df.seek(0)
df_tar = TarInfo(name="Dockerfile")
df_tar.size = len(df.buf)
tmp_tar.addfile(df_tar, fileobj=df)
tmp_tar.close()
# reset the file pointers
tmp.flush()
tmp.seek(0)
# Run the build with our tarball context
dkr = Docker()
dkr.update_image(args.tag, tmp, quiet=args.quiet)
return 0
def handle(self, *args, **kwargs):
"""
Process the command.
"""
tar = TarFile.open(datetime.today().strftime("2buntu-backup-%Y-%m-%d-%H-%M-%S.tar.bz2"), "w:bz2")
for name in self.DATABASE_MODELS:
f = MixedIO()
call_command("dumpdata", name, format="json", stdout=f)
info = TarInfo("%s.json" % name.split(".")[1])
info.size = f.rewind()
tar.addfile(info, f)
for model in self.IMAGE_MODELS:
for item in model.objects.all():
if item.image:
info = TarInfo(item.image.name)
info.size = item.image.size
tar.addfile(info, item.image)
self.stdout.write("Backup completed.")
def _dump_db(self, tar):
"""
Dump the rows in each model to the archive.
"""
# Determine the list of models to exclude
exclude = getattr(settings, 'ARCHIVE_EXCLUDE', (
'auth.Permission',
'contenttypes.ContentType',
'sessions.Session',
))
# Dump the tables to a MixedIO
data = MixedIO()
call_command('dumpdata', all=True, format='json', exclude=exclude, stdout=data)
info = TarInfo('data.json')
info.size = data.rewind()
tar.addfile(info, data)
def test_tar_compatibility(self):
# writing
db = TarDB(dirname).open('r+')
data_foo = '123'
mtime = 12345
info = TarInfo('foo')
info.mtime = mtime
db.add_record(info, data_foo)
db.close()
# reading with tarfile
import tarfile
tar = tarfile.TarFile(os.path.join(dirname, 'db00000.tar'))
info = tar.next()
data = tar.extractfile(info).read()
self.assertEqual(data, data_foo)
self.assertEqual(len(data), info.size)
self.assertEqual(info.mtime, mtime)
tar.close()
return
def tarfile(self, format, filename, content_type):
from .root.histogram import Histogram
from .combination import Combination
imgformat = "eps"
tarred_contents = StringIO()
with closing(open_tar(mode="w" + format, fileobj=tarred_contents)) as tar:
for key, context in self.resource_to_render.indexed_contexts:
if not context_renderable_as(context, imgformat):
continue
name = "/".join(map(str, key))
content = context.rendered(imgformat).content.body
info = TarInfo(name=name + "." + imgformat)
info.size = len(content)
tar.addfile(tarinfo=info, fileobj=StringIO(content))
return Response(tarred_contents.getvalue(), content_type=content_type,
content_disposition=("Content-Disposition: attachment; filename={0};"
.format(filename)))
def add_str(self, name, content, ftype, mode, mtime=None,
uid=None, gid=None, uname=None, gname=None):
'''
Add a string in memory as a file in tarball
'''
if isinstance(name, unicode):
name = name.encode("UTF-8")
ti = TarInfo(name)
# set tarinfo attribute
for v in ("name", "ftype", "mode", "mtime", "uid", "gid", "uname", "gname"):
if vars()[v] is not None:
vars(ti)[v] = vars()[v]
# set mtime to current if not specified
if mtime is None:
ti.mtime = int(time())
# unicode char is encoded in UTF-8, has changelog must be in UTF-8
if isinstance(content, unicode):
content = content.encode("UTF-8")
ti.size = len(content) if content is not None else 0
self.addfile(ti, StringIO(content))
def run(self, args, argv):
# Create a temporary tarball with our whole build context and
# dockerfile for the update
tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz")
tmp_tar = TarFile(fileobj=tmp, mode='w')
# Add the executable to the tarball
bn = os.path.basename(args.executable)
ff = "/usr/bin/%s" % bn
tmp_tar.add(args.executable, arcname=ff)
# Add any associated libraries
libs = _get_so_libs(args.executable)
if libs:
for l in libs:
tmp_tar.add(os.path.realpath(l), arcname=l)
# Create a Docker buildfile
df = StringIO()
df.write("FROM %s\n" % args.tag)
df.write("ADD . /\n")
df.seek(0)
df_tar = TarInfo(name="Dockerfile")
df_tar.size = len(df.buf)
tmp_tar.addfile(df_tar, fileobj=df)
tmp_tar.close()
# reset the file pointers
tmp.flush()
tmp.seek(0)
# Run the build with our tarball context
dkr = Docker()
dkr.update_image(args.tag, tmp, quiet=args.quiet)
return 0
