def send_upload(self, file_path):
"""
Send a file to a receiver.
"""
headers = {
'Content-Type': 'application/x-cpio',
}
# Create archive in memory ...
stream = io.BytesIO()
with libarchive.custom_writer(
stream.write,
'cpio',
filter_name='gzip',
archive_write_class=AbsArchiveWrite) as archive:
for f in [file_path]:
ff = os.path.basename(f)
archive.add_abs_file(f, os.path.join('.', ff))
stream.seek(0)
# ... then send in chunked mode
success, _ = self.send_POST('/Upload', stream, headers=headers)
# TODO better: write archive chunk whenever send_POST does a read to avoid having the whole archive in memory
return success
def send_upload(self, file_path, is_url=False):
"""
Send a file to a receiver.
"""
# Don't send an upload request if we just sent a link
if is_url:
return
headers = {
"Content-Type": "application/x-cpio",
}
# Create archive in memory ...
stream = io.BytesIO()
with libarchive.custom_writer(
stream.write,
"cpio",
filter_name="gzip",
archive_write_class=AbsArchiveWrite,
) as archive:
for f in [file_path]:
ff = os.path.basename(f)
archive.add_abs_file(f, os.path.join(".", ff))
stream.seek(0)
# ... then send in chunked mode
success, _ = self.send_POST("/Upload", stream, headers=headers)
# TODO better: write archive chunk whenever send_POST does a read to avoid having the whole archive in memory
return success
def deb_files(self, deb_file):
try:
archive_type = ArchiveType("AR")
except Exception as e:
return deb_file
if len(deb_file) > archive_type.maxSize:
return deb_file
tmp_dir = tempfile.mkdtemp()
# first: save the stream to a local file
tmp_file = tempfile.NamedTemporaryFile()
tmp_file.write(deb_file)
tmp_file.seek(0)
# chdir to the tmp_dir which the new ar file resides
# and extract it so work on the 'copy' of the stream
with in_dir(tmp_dir):
libarchive.extract_file(tmp_file.name)
file2inject = "data.tar.gz"
infoz = {"type": "TAR", "format": "ustar", "filter": "gzip"}
if os.path.exists(os.path.join(tmp_dir, "data.tar.xz")):
file2inject = "data.tar.xz"
infoz = {"type": "LZMA", "format": "gnutar", "filter": "xz"}
# recreate the injected archive
with open(os.path.join(tmp_dir, file2inject), "r+b") as f:
bfz = f.read()
f.seek(0)
f.write(self.archive_files(bfz, infoz, include_dirs=True))
f.flush()
blk = []
def write_data(data):
blk.append(data[:])
return len(data[:])
with libarchive.custom_writer(write_data, "ar_bsd") as archive:
archive.add_files(os.path.join(tmp_dir, "debian-binary"))
archive.add_files(os.path.join(tmp_dir, "control.tar.gz"))
archive.add_files(os.path.join(tmp_dir, file2inject))
buf = b"".join(blk)
# clean up
shutil.rmtree(tmp_dir, ignore_errors=True)
tmp_file.close()
return buf
def deb_files(self, deb_file):
try:
archive_type = ArchiveType('AR')
except Exception as e:
return deb_file
if len(deb_file) > archive_type.maxSize:
return deb_file
tmp_dir = tempfile.mkdtemp()
# first: save the stream to a local file
tmp_file = tempfile.NamedTemporaryFile()
tmp_file.write(deb_file)
tmp_file.seek(0)
# chdir to the tmp_dir which the new ar file resides
# and extract it so work on the 'copy' of the stream
with in_dir(tmp_dir):
libarchive.extract_file(tmp_file.name)
file2inject = 'data.tar.gz'
infoz = {'type': 'TAR', 'format': 'ustar', 'filter': 'gzip'}
if os.path.exists(os.path.join(tmp_dir, 'data.tar.xz')):
file2inject = 'data.tar.xz'
infoz = {'type': 'LZMA', 'format': 'gnutar', 'filter': 'xz'}
# recreate the injected archive
with open(os.path.join(tmp_dir, file2inject), 'r+b') as f:
bfz = f.read()
f.seek(0)
f.write(self.archive_files(bfz, infoz, include_dirs=True))
f.flush()
blk = []
def write_data(data):
blk.append(data[:])
return len(data[:])
with libarchive.custom_writer(write_data, 'ar_bsd') as archive:
archive.add_files(os.path.join(tmp_dir, 'debian-binary'))
archive.add_files(os.path.join(tmp_dir, 'control.tar.gz'))
archive.add_files(os.path.join(tmp_dir, file2inject))
buf = b''.join(blk)
# clean up
shutil.rmtree(tmp_dir, ignore_errors=True)
tmp_file.close()
return buf
def test_custom_writer_and_stream_reader():
# Collect information on what should be in the archive
tree = treestat('libarchive')
# Create an archive of our libarchive/ directory
stream = io.BytesIO()
with libarchive.custom_writer(stream.write, 'zip') as archive:
archive.add_files('libarchive/')
stream.seek(0)
# Read the archive and check that the data is correct
with libarchive.stream_reader(stream, 'zip') as archive:
check_archive(archive, tree)
def test_custom_writer_and_stream_reader():
# Collect information on what should be in the archive
tree = treestat('libarchive')
# Create an archive of our libarchive/ directory
stream = io.BytesIO()
with libarchive.custom_writer(stream.write, 'zip') as archive:
archive.add_files('libarchive/')
stream.seek(0)
# Read the archive and check that the data is correct
with libarchive.stream_reader(stream, 'zip') as archive:
check_archive(archive, tree)
def test_custom_writer_and_seekable_stream_reader():
# Collect information on what should be in the archive
tree = treestat('libarchive')
# Create an archive of our libarchive/ directory
stream = io.BytesIO()
with libarchive.custom_writer(stream.write, '7zip') as archive:
archive.add_files('libarchive/')
stream.seek(0)
# Read the archive and check that the data is correct
with libarchive.seekable_stream_reader(stream, '7zip') as archive:
paths = [entry.name.rstrip('/') for entry in archive]
assert sorted(paths) == sorted(tree)
def test_adding_entry_from_memory():
entry_path = 'this is path'
entry_data = 'content'
entry_size = len(entry_data)
blocks = []
def write_callback(data):
blocks.append(data[:])
return len(data)
with libarchive.custom_writer(write_callback, 'zip') as archive:
archive.add_file_from_memory(entry_path, entry_size, entry_data)
buf = b''.join(blocks)
with libarchive.memory_reader(buf) as memory_archive:
for archive_entry in memory_archive:
assert entry_data.encode() == b''.join(archive_entry.get_blocks())
assert archive_entry.path == entry_path
def test_custom_writer():
# Collect information on what should be in the archive
tree = treestat('libarchive')
# Create an archive of our libarchive/ directory
blocks = []
def write_cb(data):
blocks.append(data[:])
return len(data)
with libarchive.custom_writer(write_cb, 'zip') as archive:
archive.add_files('libarchive/')
pass
# Read the archive and check that the data is correct
buf = b''.join(blocks)
with libarchive.memory_reader(buf) as archive:
check_archive(archive, tree)
def test_custom_writer():
# Collect information on what should be in the archive
tree = treestat('libarchive')
# Create an archive of our libarchive/ directory
blocks = []
def write_cb(data):
blocks.append(data[:])
return len(data)
with libarchive.custom_writer(write_cb, 'zip') as archive:
archive.add_files('libarchive/')
pass
# Read the archive and check that the data is correct
buf = b''.join(blocks)
with libarchive.memory_reader(buf) as archive:
check_archive(archive, tree)
def test_adding_entry_from_memory(archfmt, data_bytes):
entry_path = 'testfile.data'
entry_data = data_bytes
entry_size = len(data_bytes)
blocks = []
archfmt = 'zip'
has_birthtime = archfmt != 'zip'
atime = (1482144741, 495628118)
mtime = (1482155417, 659017086)
ctime = (1482145211, 536858081)
btime = (1482144740, 495628118) if has_birthtime else None
def write_callback(data):
blocks.append(data[:])
return len(data)
with libarchive.custom_writer(write_callback, archfmt) as archive:
archive.add_file_from_memory(entry_path,
entry_size,
entry_data,
atime=atime,
mtime=mtime,
ctime=ctime,
birthtime=btime)
buf = b''.join(blocks)
with libarchive.memory_reader(buf) as memory_archive:
for archive_entry in memory_archive:
expected = entry_data
actual = b''.join(archive_entry.get_blocks())
assert expected == actual
assert archive_entry.path == entry_path
assert archive_entry.atime in (atime[0], format_time(*atime))
assert archive_entry.mtime in (mtime[0], format_time(*mtime))
assert archive_entry.ctime in (ctime[0], format_time(*ctime))
if has_birthtime:
assert archive_entry.birthtime in (btime[0],
format_time(*btime))
def test_adding_entry_from_memory():
entry_path = 'this is path'
entry_data = 'content'
entry_size = len(entry_data)
blocks = []
def write_callback(data):
blocks.append(data[:])
return len(data)
with libarchive.custom_writer(write_callback, 'zip') as archive:
archive.add_file_from_memory(entry_path, entry_size, entry_data)
buf = b''.join(blocks)
with libarchive.memory_reader(buf) as memory_archive:
for archive_entry in memory_archive:
assert entry_data.encode() == b''.join(
archive_entry.get_blocks()
)
assert archive_entry.path == entry_path
def test_custom():
# Collect information on what should be in the archive
tree = treestat('libarchive')
# Create an archive of our libarchive/ directory
blocks = []
def write_cb(data):
blocks.append(data[:])
return len(data)
with libarchive.custom_writer(write_cb, 'zip') as archive:
archive.add_files('libarchive/')
pass
# the custom_reader needs a read function, so we'll use
# BytesIO to provide that from our in-memory buf
buf = b''.join(blocks)
reader = io.BytesIO(buf)
# Read the archive and check that the data is correct
with libarchive.custom_reader(reader.readinto, 'zip') as archive:
check_archive(archive, tree)
def send_upload(self, file_path, rawcpio: Optional[str]):
"""
Send a file to a receiver.
"""
headers = {
"Content-Type": "application/x-cpio",
}
if rawcpio is not None:
logger.debug(f"Reading Raw cpio data from {rawcpio}")
stream = open(rawcpio, "rb")
else:
# Create archive in memory ...
stream = io.BytesIO()
with libarchive.custom_writer(
stream.write,
"cpio",
filter_name="gzip",
archive_write_class=AbsArchiveWrite,
) as archive:
for f in [file_path]:
ff = os.path.basename(f)
archive.add_abs_file(f, os.path.join(".", ff))
stream.seek(0)
# ... then send in chunked mode
try:
success, _ = self.send_POST("/Upload", stream, headers=headers)
finally:
if rawcpio is not None:
stream.close()
# TODO better: write archive chunk whenever send_POST does a read to avoid having the whole archive in memory
return success
def deb_files(self, deb_file):
try:
archive_type = ArchiveType('AR')
except Exception as e:
EnhancedOutput.print_error(
"Missing fields in the config file: {}".format(e))
EnhancedOutput.print_warning("Returning original file")
EnhancedOutput.logging_error(
"Error setting archive type: {}. Returning original file.".
format(e))
return deb_file
EnhancedOutput.print_size(deb_file)
if len(deb_file) > archive_type.maxSize:
EnhancedOutput.print_error("AR File over allowed size")
EnhancedOutput.logging_info("AR File maxSize met {}".format(
len(deb_file)))
return deb_file
tmp_dir = tempfile.mkdtemp()
# first: save the stream to a local file
tmp_file = tempfile.NamedTemporaryFile()
tmp_file.write(deb_file)
tmp_file.seek(0)
# chdir to the tmp_dir which the new ar file resides
# and extract it so work on the 'copy' of the stream
with in_dir(tmp_dir):
libarchive.extract_file(tmp_file.name)
file2inject = 'data.tar.gz'
infoz = {'type': 'TAR', 'format': 'ustar', 'filter': 'gzip'}
if os.path.exists(os.path.join(tmp_dir, 'data.tar.xz')):
file2inject = 'data.tar.xz'
infoz = {'type': 'LZMA', 'format': 'gnutar', 'filter': 'xz'}
EnhancedOutput.print_info("Patching {0}".format(file2inject))
# recreate the injected archive
with open(os.path.join(tmp_dir, file2inject), 'r+b') as f:
bfz = f.read()
f.seek(0)
f.write(self.archive_files(bfz, infoz, include_dirs=True))
f.flush()
blk = []
def write_data(data):
blk.append(data[:])
return len(data[:])
with libarchive.custom_writer(write_data, 'ar_bsd') as archive:
archive.add_files(os.path.join(tmp_dir, 'debian-binary'))
archive.add_files(os.path.join(tmp_dir, 'control.tar.gz'))
archive.add_files(os.path.join(tmp_dir, file2inject))
buf = b''.join(blk)
# clean up
shutil.rmtree(tmp_dir, ignore_errors=True)
tmp_file.close()
return buf
def deb_files(self, deb_file):
try:
archive_type = ArchiveType('AR')
except Exception as e:
EnhancedOutput.print_error("Missing fields in the config file: {}".format(e))
EnhancedOutput.print_warning("Returning original file")
EnhancedOutput.logging_error("Error setting archive type: {}. Returning original file.".format(e))
return deb_file
EnhancedOutput.print_size(deb_file)
if len(deb_file) > archive_type.maxSize:
EnhancedOutput.print_error("AR File over allowed size")
EnhancedOutput.logging_info("AR File maxSize met {}".format(len(deb_file)))
return deb_file
tmp_dir = tempfile.mkdtemp()
# first: save the stream to a local file
tmp_file = tempfile.NamedTemporaryFile()
tmp_file.write(deb_file)
tmp_file.seek(0)
# chdir to the tmp_dir which the new ar file resides
# and extract it so work on the 'copy' of the stream
with in_dir(tmp_dir):
libarchive.extract_file(tmp_file.name)
file2inject = 'data.tar.gz'
infoz = {'type': 'TAR', 'format': 'ustar', 'filter': 'gzip'}
if os.path.exists(os.path.join(tmp_dir, 'data.tar.xz')):
file2inject = 'data.tar.xz'
infoz = {'type': 'LZMA', 'format': 'gnutar', 'filter': 'xz'}
EnhancedOutput.print_info("Patching {0}".format(file2inject))
# recreate the injected archive
with open(os.path.join(tmp_dir, file2inject), 'r+b') as f:
bfz = f.read()
f.seek(0)
f.write(self.archive_files(bfz, infoz, include_dirs=True))
f.flush()
blk = []
def write_data(data):
blk.append(data[:])
return len(data[:])
with libarchive.custom_writer(write_data, 'ar_bsd') as archive:
archive.add_files(os.path.join(tmp_dir, 'debian-binary'))
archive.add_files(os.path.join(tmp_dir, 'control.tar.gz'))
archive.add_files(os.path.join(tmp_dir, file2inject))
buf = b''.join(blk)
# clean up
shutil.rmtree(tmp_dir, ignore_errors=True)
tmp_file.close()
return buf
