def unzip(data: bytes) -> bytes:
if data[:2] == b"VZ":
if data[-2:] != b"zv":
raise RuntimeError(f"VZ: Invalid footer: {data[-2:]!r}")
if data[2:3] != b"a":
raise RuntimeError(f"VZ: Invalid version: {data[2:3]!r}")
filters = (lzma._decode_filter_properties(lzma.FILTER_LZMA1,
data[7:12]),
) # type: ignore
decompressor = lzma.LZMADecompressor(lzma.FORMAT_RAW, filters=filters)
checksum, decompressed_size = struct.unpack("<II", data[-10:-2])
data = decompressor.decompress(data[12:-9],
max_length=decompressed_size)
if crc32(data) != checksum:
raise RuntimeError(
"VZ: CRC32 checksum doesn't match for decompressed data")
else:
try:
with ZipFile(BytesIO(data)) as zf:
data = zf.read(zf.filelist[0])
except BadZipFile:
pass
return data
def __init__(self, coders: List[Dict[str, Any]], size: int, crc: Optional[int]) -> None:
self.input_size = size
self.consumed = 0 # type: int
self.crc = crc
self.digest = None # type: Optional[int]
filters = [] # type: List[Dict[str, Any]]
try:
for coder in coders:
if coder['numinstreams'] != 1 or coder['numoutstreams'] != 1:
raise UnsupportedCompressionMethodError('Only a simple compression method is currently supported.')
filter = self.lzma_methods_map.get(coder['method'], None)
if filter is not None:
properties = coder.get('properties', None)
if properties is not None:
filters[:0] = [lzma._decode_filter_properties(filter, properties)] # type: ignore
else:
filters[:0] = [{'id': filter}]
else:
raise UnsupportedCompressionMethodError
except UnsupportedCompressionMethodError as e:
filter = self.alt_methods_map.get(coders[0]['method'], None)
if len(coders) == 1 and filter is not None:
if filter == self.FILTER_BZIP2:
self.decompressor = bz2.BZ2Decompressor() # type: Union[bz2.BZ2Decompressor, lzma.LZMADecompressor]
else:
raise e
self.can_partial_decompress = False
else:
raise e
else:
self.decompressor = lzma.LZMADecompressor(format=lzma.FORMAT_RAW, filters=filters)
self.can_partial_decompress = True
self.filters = filters
def _get_lzma_decompressor(self, coders: List[Dict[str, Any]],
unpacksize: int):
filters = [] # type: List[Dict[str, Any]]
lzma1 = False
for coder in coders:
if coder['numinstreams'] != 1 or coder['numoutstreams'] != 1:
raise UnsupportedCompressionMethodError(
'Only a simple compression method is currently supported.')
if not SupportedMethods.is_native_coder(coder):
raise UnsupportedCompressionMethodError
properties = coder.get('properties', None)
filter_id = SupportedMethods.get_filter_id(coder)
if filter_id == FILTER_LZMA:
lzma1 = True
if properties is not None:
filters[:0] = [
lzma._decode_filter_properties(filter_id, properties)
] # type: ignore
else:
filters[:0] = [{'id': filter_id}]
if lzma1:
return LZMA1Decompressor(filters, unpacksize)
else:
return lzma.LZMADecompressor(format=lzma.FORMAT_RAW,
filters=filters)
def _init(self):
props = lzma._encode_filter_properties(self.compress_options)
self._comp = lzma.LZMACompressor(
lzma.FORMAT_RAW,
filters=(
lzma._decode_filter_properties(self.compress_options["id"], props),))
return struct.pack("<BBH", 9, 4, len(props)) + props
def decompress_lump(reader: ByteIO) -> ByteIO:
magic = reader.read_fourcc()
assert magic == 'LZMA', f'Invalid LZMA compressed header: {magic}'
decompressed_size = reader.read_uint32()
compressed_size = reader.read_uint32()
filter_properties = lzma._decode_filter_properties(
lzma.FILTER_LZMA1, reader.read(5))
compressed_buffer = reader.read(compressed_size)
decompressed_buffer = bytearray()
while True:
decompressor = lzma.LZMADecompressor(lzma.FORMAT_RAW,
filters=(filter_properties, ))
try:
result = decompressor.decompress(compressed_buffer)
except lzma.LZMAError:
if not decompressed_buffer:
raise # Error on the first iteration; bail out.
break # Leftover data is not a valid LZMA/XZ stream; ignore it.
decompressed_buffer.extend(result)
compressed_buffer = decompressor.unused_data
if not compressed_buffer:
break
assert decompressor.eof, 'Compressed data ended before the end-of-stream marker was reached'
decompressed_buffer = decompressed_buffer[:decompressed_size]
assert decompressed_size == len(
decompressed_buffer
), 'Decompressed data does not match the expected size'
return ByteIO(decompressed_buffer)
def decompress_valve_LZMA(data: bytes) -> bytes:
"""valve LZMA header adapter"""
magic, true_size, compressed_size, properties = struct.unpack(
"4s2I5s", data[:17])
_filter = lzma._decode_filter_properties(lzma.FILTER_LZMA1, properties)
decompressor = lzma.LZMADecompressor(lzma.FORMAT_RAW, None, [_filter])
decompressed_data = decompressor.decompress(data[17:17 + compressed_size])
return decompressed_data[:true_size] # trim any excess bytes
def process_chunk(chunk, depot_key): decrypted_chunk = CryptoUtil.symmetric_decrypt(chunk, depot_key) if decrypted_chunk[:2] == 'VZ': filter = lzma._decode_filter_properties(lzma.FILTER_LZMA1, decrypted_chunk[7:12]) lzmadec = lzma.LZMADecompressor(lzma.FORMAT_RAW, None, [filter]) return lzmadec.decompress(decrypted_chunk[12:len(decrypted_chunk)-10]) else: zip_buffer = StringIO.StringIO(decrypted_chunk) with zipfile.ZipFile(zip_buffer, 'r') as zip: return zip.read(zip.namelist()[0])
def process_chunk(chunk, depot_key): decrypted_chunk = CryptoUtil.symmetric_decrypt(chunk, depot_key) if decrypted_chunk[:2] == 'VZ': filter = lzma._decode_filter_properties(lzma.FILTER_LZMA1, decrypted_chunk[7:12]) lzmadec = lzma.LZMADecompressor(lzma.FORMAT_RAW, None, [filter]) return lzmadec.decompress(decrypted_chunk[12:len(decrypted_chunk)-10]) else: zip_buffer = StringIO.StringIO(decrypted_chunk) with zipfile.ZipFile(zip_buffer, 'r') as zip: return zip.read(zip.namelist()[0])
def decompress(self, data):
if self._decomp is None:
if len(self._unconsumed) <= 4:
return b''
(psize,) = struct.unpack('<H', self._unconsumed[2:4])
if len(self._unconsumed) <= 4 + psize:
return b''
self._decomp = lzma.LZMADecompressor(lzma.FORMAT_RAW, filters=[lzma._decode_filter_properties(lzma.FILTER_LZMA1, self._unconsumed[4:4 + psize])])
data = self._unconsumed[4 + psize:]
del self._unconsumed
result = self._decomp.decompress(data)
self.eof = self._decomp.eof
return result
def _set_lzma_decompressor(self, coders: List[Dict[str, Any]]):
filters = [] # type: List[Dict[str, Any]]
for coder in coders:
filter = self.lzma_methods_map.get(coder['method'], None)
if filter is not None:
properties = coder.get('properties', None)
if properties is not None:
filters[:0] = [
lzma._decode_filter_properties(filter, properties)
] # type: ignore
else:
filters[:0] = [{'id': filter}]
else:
raise UnsupportedCompressionMethodError
return lzma.LZMADecompressor(format=lzma.FORMAT_RAW, filters=filters)
def get_chunk(self, app_id, depot_id, chunk_id):
"""Download a single content chunk
:param app_id: App ID
:type app_id: int
:param depot_id: Depot ID
:type depot_id: int
:param chunk_id: Chunk ID
:type chunk_id: int
:returns: chunk data
:rtype: bytes
:raises SteamError: error message
"""
if (depot_id, chunk_id) not in self._chunk_cache:
resp = self.cdn_cmd('depot', '%s/chunk/%s' % (depot_id, chunk_id))
data = symmetric_decrypt(resp.content,
self.get_depot_key(app_id, depot_id))
if data[:2] == b'VZ':
if data[-2:] != b'zv':
raise SteamError("VZ: Invalid footer: %s" %
repr(data[-2:]))
if data[2:3] != b'a':
raise SteamError("VZ: Invalid version: %s" %
repr(data[2:3]))
vzfilter = lzma._decode_filter_properties(
lzma.FILTER_LZMA1, data[7:12])
vzdec = lzma.LZMADecompressor(lzma.FORMAT_RAW,
filters=[vzfilter])
checksum, decompressed_size = struct.unpack(
'<II', data[-10:-2])
# decompress_size is needed since lzma will sometime produce longer output
# [12:-9] is need as sometimes lzma will produce shorter output
# together they get us the right data
data = vzdec.decompress(data[12:-9])[:decompressed_size]
if crc32(data) != checksum:
raise SteamError(
"VZ: CRC32 checksum doesn't match for decompressed data"
)
else:
with ZipFile(BytesIO(data)) as zf:
data = zf.read(zf.filelist[0])
self._chunk_cache[(depot_id, chunk_id)] = data
return self._chunk_cache[(depot_id, chunk_id)]
def decompress_lump(reader: ByteIO):
lzma_id = reader.read_fourcc()
assert lzma_id == "LZMA", f"Unknown compressed header({lzma_id})"
decompressed_size = reader.read_uint32()
compressed_size = reader.read_uint32()
filters = lzma._decode_filter_properties(lzma.FILTER_LZMA1,
reader.read(5))
decompressor = lzma.LZMADecompressor(format=lzma.FORMAT_RAW,
filters=[filters])
decompressed = decompressor.decompress(reader.read(compressed_size))
if len(decompressed) > decompressed_size:
decompressed = decompressed[:decompressed_size]
new_reader = ByteIO(decompressed)
assert new_reader.size(
) == decompressed_size, 'Compressed lump size does not match expected'
return new_reader
def _set_lzma_decompressor(self, coders: List[Dict[str, Any]]) -> None:
filters = [] # type: List[Dict[str, Any]]
for coder in coders:
if coder['numinstreams'] != 1 or coder['numoutstreams'] != 1:
raise UnsupportedCompressionMethodError(
'Only a simple compression method is currently supported.')
filter_id = self.lzma_methods_map.get(coder['method'], None)
if filter_id is None:
raise UnsupportedCompressionMethodError
properties = coder.get('properties', None)
if properties is not None:
filters[:0] = [
lzma._decode_filter_properties(filter_id, properties)
] # type: ignore
else:
filters[:0] = [{'id': filter_id}]
self.decompressor = lzma.LZMADecompressor(
format=lzma.FORMAT_RAW, filters=filters
) # type: Union[bz2.BZ2Decompressor, lzma.LZMADecompressor, AESDecompressor, CopyDecompressor] # noqa
def __init__(self, s: bytes):
data = struct.unpack("<4s I I 5s", s[:17])
dic_size = struct.unpack("<I", s[12 + 1:12 + 1 + 4])
dic_size = dic_size[0]
d = struct.unpack("B", s[12:12 + 1])
d = d[0]
lc = d % 9
d = d // 9
pb = d // 5
lp = d % 5
filter_1 = lzma._decode_filter_properties(lzma.FILTER_LZMA1, data[3])
filter_2 = {
"id": lzma.FILTER_LZMA1,
"pb": pb,
"lp": lp,
"lc": lc,
"dict_size": dic_size
}
data = lzma.decompress(s[17:], lzma.FORMAT_RAW, filters=[filter_2])
a = 1
def _lzma(f, size):
import lzma
data = f.read()
props = lzma._decode_filter_properties(lzma.FILTER_LZMA1, data[0:5])
return lzma.decompress(data[5:], lzma.FORMAT_RAW, filters=[props])
def _init(self):
props = lzma._encode_filter_properties({'id': lzma.FILTER_LZMA1})
self._comp = lzma.LZMACompressor(lzma.FORMAT_RAW, filters=[lzma._decode_filter_properties(lzma.FILTER_LZMA1, props)])
return struct.pack('<BBH', 9, 4, len(props)) + props
def update_event(self, inp=-1):
self.set_output_val(0, lzma._decode_filter_properties(self.input(0), self.input(1)))
parser.add_argument(
"-rawlzma1",
action='store_true',
help='Use Raw LZMA version 1 (default: XZ compression mode)')
parser.add_argument("-rawarm",
action='store_true',
help='Use Raw LZMA ARM (default: XZ compression mode)')
args = parser.parse_args()
binary_object1 = open(args.input, 'rb').read()
if args.lzma1 is True:
props = lzma._encode_filter_properties({'id': lzma.FILTER_LZMA1})
lzma_comp = lzma.LZMACompressor(
lzma.FORMAT_ALONE,
filters=[lzma._decode_filter_properties(lzma.FILTER_LZMA1, props)],
preset=args.preset)
binary_object2 = lzma_comp.compress(binary_object1)
lzma_comp.flush()
elif args.lzma2 is True:
props = lzma._encode_filter_properties({'id': lzma.FILTER_LZMA2})
lzma_comp = lzma.LZMACompressor(
lzma.FORMAT_RAW,
filters=[lzma._decode_filter_properties(lzma.FILTER_LZMA2, props)],
preset=args.preset)
binary_object2 = lzma_comp.compress(binary_object1)
lzma_comp.flush()
elif args.rawlzma1 is True:
props = lzma._encode_filter_properties({'id': lzma.FILTER_LZMA1})
lzma_comp = lzma.LZMACompressor(
lzma.FORMAT_RAW,
