def test_lzma2():
raw = P('template-functions.json', allow_user_override=False, data=True)
ibuf, obuf = BytesIO(raw), BytesIO()
props = lzma.compress(ibuf.read, obuf.write, False)
cc = obuf.getvalue()
ibuf, obuf = BytesIO(cc), BytesIO()
LZMA2Filter(props, 0, 1)(ibuf, obuf)
if obuf.getvalue() != raw:
raise ValueError('Roundtripping via LZMA2 failed')
def test_lzma2():
raw = P('template-functions.json', allow_user_override=False, data=True)
ibuf, obuf = BytesIO(raw), BytesIO()
props = lzma.compress(ibuf.read, obuf.write, False)
cc = obuf.getvalue()
ibuf, obuf = BytesIO(cc), BytesIO()
LZMA2Filter(props, 0, 1)(ibuf, obuf)
if obuf.getvalue() != raw:
raise ValueError('Roundtripping via LZMA2 failed')
def compress(raw, outfile=None, level=5, check_type='crc64'):
'''
Compress the specified data into a .xz stream (which can be written directly as
an .xz file.
:param raw: A bytestring or a file-like object open for reading
:outfile: A file like object open for writing.
The .xz stream is written into it. If not specified then a SpooledTemporaryFile
is created and returned by this function.
:level: An integer between 0 and 9 with 0 being fastest/worst compression and 9 being
slowest/best compression
:check_type: The type of data integrity check to write into the output .xz stream.
Should be one of: 'crc32', 'crc64', 'sha256', or None
'''
if isinstance(raw, bytes):
raw = BytesIO(raw)
outfile = outfile or SpooledTemporaryFile(50 * 1024 * 1024,
'_xz_decompress')
# Write stream header
outfile.write(HEADER_MAGIC)
check_type = {
'crc': 1,
'crc32': 1,
'sha256': 0xa,
None: 0,
'': 0,
'none': 0,
'None': 0
}.get(check_type, 4)
stream_flags = as_bytes(0, check_type)
outfile.write(stream_flags)
outfile.write(pack(b'<I', crc32(stream_flags)))
# Write block header
filter_flags = encode_var_int(LZMA2_FILTER_ID) + encode_var_int(
1) + lzma.preset_map[level]
block_header = align(b'\0\0' + filter_flags)
bhs = ((4 + len(block_header)) // 4) - 1
block_header = as_bytes(bhs) + block_header[1:]
block_header += pack(b'<I', crc32(block_header))
start = outfile.tell()
outfile.write(block_header)
# Write compressed data and check
checker = {
0: DummyChecker,
1: CRCChecker,
4: CRCChecker,
0xa: Sha256Checker
}[check_type](check_type)
uncompressed_size = [0]
def read(n):
ans = raw.read(n)
if ans:
uncompressed_size[0] += len(ans)
checker(ans)
return ans
lzma.compress(read, outfile.write, None, level)
unpadded_size = outfile.tell() - start
pos = outfile.tell()
if pos % 4:
outfile.write(b'\0' * (4 - (pos % 4)))
checker.finish()
if check_type:
cc = checker.code_as_bytes
outfile.write(cc)
unpadded_size += len(cc)
# Write index
index = b'\0' + encode_var_int(1)
index += encode_var_int(unpadded_size) + encode_var_int(
uncompressed_size[0])
if len(index) % 4:
index += b'\0' * (4 - len(index) % 4)
outfile.write(index), outfile.write(pack(b'<I', crc32(index)))
# Write stream footer
backwards_size = pack(b'<I', ((len(index) + 4) // 4) - 1)
outfile.write(pack(b'<I', crc32(backwards_size + stream_flags)))
outfile.write(backwards_size), outfile.write(stream_flags), outfile.write(
FOOTER_MAGIC)
def compress(raw, outfile=None, level=5, check_type='crc64'):
'''
Compress the specified data into a .xz stream (which can be written directly as
an .xz file.
:param raw: A bytestring or a file-like object open for reading
:outfile: A file like object open for writing.
The .xz stream is written into it. If not specified then a SpooledTemporaryFile
is created and returned by this function.
:level: An integer between 0 and 9 with 0 being fastest/worst compression and 9 being
slowest/best compression
:check_type: The type of data integrity check to write into the output .xz stream.
Should be one of: 'crc32', 'crc64', 'sha256', or None
'''
if isinstance(raw, bytes):
raw = BytesIO(raw)
outfile = outfile or SpooledTemporaryFile(50 * 1024 * 1024, '_xz_decompress')
# Write stream header
outfile.write(HEADER_MAGIC)
check_type = {
'crc': 1,
'crc32': 1,
'sha256': 0xa,
None: 0,
'': 0,
'none': 0,
'None': 0
}.get(check_type, 4)
stream_flags = as_bytes(0, check_type)
outfile.write(stream_flags)
outfile.write(pack(b'<I', crc32(stream_flags)))
# Write block header
filter_flags = encode_var_int(LZMA2_FILTER_ID
) + encode_var_int(1) + lzma.preset_map[level]
block_header = align(b'\0\0' + filter_flags)
bhs = ((4 + len(block_header)) // 4) - 1
block_header = as_bytes(bhs) + block_header[1:]
block_header += pack(b'<I', crc32(block_header))
start = outfile.tell()
outfile.write(block_header)
# Write compressed data and check
checker = {
0: DummyChecker,
1: CRCChecker,
4: CRCChecker,
0xa: Sha256Checker
}[check_type](check_type)
uncompressed_size = [0]
def read(n):
ans = raw.read(n)
if ans:
uncompressed_size[0] += len(ans)
checker(ans)
return ans
lzma.compress(read, outfile.write, None, level)
unpadded_size = outfile.tell() - start
pos = outfile.tell()
if pos % 4:
outfile.write(b'\0' * (4 - (pos % 4)))
checker.finish()
if check_type:
cc = checker.code_as_bytes
outfile.write(cc)
unpadded_size += len(cc)
# Write index
index = b'\0' + encode_var_int(1)
index += encode_var_int(unpadded_size) + encode_var_int(uncompressed_size[0])
if len(index) % 4:
index += b'\0' * (4 - len(index) % 4)
outfile.write(index), outfile.write(pack(b'<I', crc32(index)))
# Write stream footer
backwards_size = pack(b'<I', ((len(index) + 4) // 4) - 1)
outfile.write(pack(b'<I', crc32(backwards_size + stream_flags)))
outfile.write(backwards_size), outfile.write(stream_flags
), outfile.write(FOOTER_MAGIC)