def unzip(data: bytes) -> bytes: if data[:2] == b"VZ": if data[-2:] != b"zv": raise RuntimeError(f"VZ: Invalid footer: {data[-2:]!r}") if data[2:3] != b"a": raise RuntimeError(f"VZ: Invalid version: {data[2:3]!r}") filters = (lzma._decode_filter_properties(lzma.FILTER_LZMA1, data[7:12]), ) # type: ignore decompressor = lzma.LZMADecompressor(lzma.FORMAT_RAW, filters=filters) checksum, decompressed_size = struct.unpack("<II", data[-10:-2]) data = decompressor.decompress(data[12:-9], max_length=decompressed_size) if crc32(data) != checksum: raise RuntimeError( "VZ: CRC32 checksum doesn't match for decompressed data") else: try: with ZipFile(BytesIO(data)) as zf: data = zf.read(zf.filelist[0]) except BadZipFile: pass return data
def __init__(self, coders: List[Dict[str, Any]], size: int, crc: Optional[int]) -> None: self.input_size = size self.consumed = 0 # type: int self.crc = crc self.digest = None # type: Optional[int] filters = [] # type: List[Dict[str, Any]] try: for coder in coders: if coder['numinstreams'] != 1 or coder['numoutstreams'] != 1: raise UnsupportedCompressionMethodError('Only a simple compression method is currently supported.') filter = self.lzma_methods_map.get(coder['method'], None) if filter is not None: properties = coder.get('properties', None) if properties is not None: filters[:0] = [lzma._decode_filter_properties(filter, properties)] # type: ignore else: filters[:0] = [{'id': filter}] else: raise UnsupportedCompressionMethodError except UnsupportedCompressionMethodError as e: filter = self.alt_methods_map.get(coders[0]['method'], None) if len(coders) == 1 and filter is not None: if filter == self.FILTER_BZIP2: self.decompressor = bz2.BZ2Decompressor() # type: Union[bz2.BZ2Decompressor, lzma.LZMADecompressor] else: raise e self.can_partial_decompress = False else: raise e else: self.decompressor = lzma.LZMADecompressor(format=lzma.FORMAT_RAW, filters=filters) self.can_partial_decompress = True self.filters = filters
def _get_lzma_decompressor(self, coders: List[Dict[str, Any]], unpacksize: int): filters = [] # type: List[Dict[str, Any]] lzma1 = False for coder in coders: if coder['numinstreams'] != 1 or coder['numoutstreams'] != 1: raise UnsupportedCompressionMethodError( 'Only a simple compression method is currently supported.') if not SupportedMethods.is_native_coder(coder): raise UnsupportedCompressionMethodError properties = coder.get('properties', None) filter_id = SupportedMethods.get_filter_id(coder) if filter_id == FILTER_LZMA: lzma1 = True if properties is not None: filters[:0] = [ lzma._decode_filter_properties(filter_id, properties) ] # type: ignore else: filters[:0] = [{'id': filter_id}] if lzma1: return LZMA1Decompressor(filters, unpacksize) else: return lzma.LZMADecompressor(format=lzma.FORMAT_RAW, filters=filters)
def _init(self): props = lzma._encode_filter_properties(self.compress_options) self._comp = lzma.LZMACompressor( lzma.FORMAT_RAW, filters=( lzma._decode_filter_properties(self.compress_options["id"], props),)) return struct.pack("<BBH", 9, 4, len(props)) + props
def decompress_lump(reader: ByteIO) -> ByteIO: magic = reader.read_fourcc() assert magic == 'LZMA', f'Invalid LZMA compressed header: {magic}' decompressed_size = reader.read_uint32() compressed_size = reader.read_uint32() filter_properties = lzma._decode_filter_properties( lzma.FILTER_LZMA1, reader.read(5)) compressed_buffer = reader.read(compressed_size) decompressed_buffer = bytearray() while True: decompressor = lzma.LZMADecompressor(lzma.FORMAT_RAW, filters=(filter_properties, )) try: result = decompressor.decompress(compressed_buffer) except lzma.LZMAError: if not decompressed_buffer: raise # Error on the first iteration; bail out. break # Leftover data is not a valid LZMA/XZ stream; ignore it. decompressed_buffer.extend(result) compressed_buffer = decompressor.unused_data if not compressed_buffer: break assert decompressor.eof, 'Compressed data ended before the end-of-stream marker was reached' decompressed_buffer = decompressed_buffer[:decompressed_size] assert decompressed_size == len( decompressed_buffer ), 'Decompressed data does not match the expected size' return ByteIO(decompressed_buffer)
def decompress_valve_LZMA(data: bytes) -> bytes: """valve LZMA header adapter""" magic, true_size, compressed_size, properties = struct.unpack( "4s2I5s", data[:17]) _filter = lzma._decode_filter_properties(lzma.FILTER_LZMA1, properties) decompressor = lzma.LZMADecompressor(lzma.FORMAT_RAW, None, [_filter]) decompressed_data = decompressor.decompress(data[17:17 + compressed_size]) return decompressed_data[:true_size] # trim any excess bytes
def process_chunk(chunk, depot_key): decrypted_chunk = CryptoUtil.symmetric_decrypt(chunk, depot_key) if decrypted_chunk[:2] == 'VZ': filter = lzma._decode_filter_properties(lzma.FILTER_LZMA1, decrypted_chunk[7:12]) lzmadec = lzma.LZMADecompressor(lzma.FORMAT_RAW, None, [filter]) return lzmadec.decompress(decrypted_chunk[12:len(decrypted_chunk)-10]) else: zip_buffer = StringIO.StringIO(decrypted_chunk) with zipfile.ZipFile(zip_buffer, 'r') as zip: return zip.read(zip.namelist()[0])
def process_chunk(chunk, depot_key): decrypted_chunk = CryptoUtil.symmetric_decrypt(chunk, depot_key) if decrypted_chunk[:2] == 'VZ': filter = lzma._decode_filter_properties(lzma.FILTER_LZMA1, decrypted_chunk[7:12]) lzmadec = lzma.LZMADecompressor(lzma.FORMAT_RAW, None, [filter]) return lzmadec.decompress(decrypted_chunk[12:len(decrypted_chunk)-10]) else: zip_buffer = StringIO.StringIO(decrypted_chunk) with zipfile.ZipFile(zip_buffer, 'r') as zip: return zip.read(zip.namelist()[0])
def decompress(self, data): if self._decomp is None: if len(self._unconsumed) <= 4: return b'' (psize,) = struct.unpack('<H', self._unconsumed[2:4]) if len(self._unconsumed) <= 4 + psize: return b'' self._decomp = lzma.LZMADecompressor(lzma.FORMAT_RAW, filters=[lzma._decode_filter_properties(lzma.FILTER_LZMA1, self._unconsumed[4:4 + psize])]) data = self._unconsumed[4 + psize:] del self._unconsumed result = self._decomp.decompress(data) self.eof = self._decomp.eof return result
def _set_lzma_decompressor(self, coders: List[Dict[str, Any]]): filters = [] # type: List[Dict[str, Any]] for coder in coders: filter = self.lzma_methods_map.get(coder['method'], None) if filter is not None: properties = coder.get('properties', None) if properties is not None: filters[:0] = [ lzma._decode_filter_properties(filter, properties) ] # type: ignore else: filters[:0] = [{'id': filter}] else: raise UnsupportedCompressionMethodError return lzma.LZMADecompressor(format=lzma.FORMAT_RAW, filters=filters)
def get_chunk(self, app_id, depot_id, chunk_id): """Download a single content chunk :param app_id: App ID :type app_id: int :param depot_id: Depot ID :type depot_id: int :param chunk_id: Chunk ID :type chunk_id: int :returns: chunk data :rtype: bytes :raises SteamError: error message """ if (depot_id, chunk_id) not in self._chunk_cache: resp = self.cdn_cmd('depot', '%s/chunk/%s' % (depot_id, chunk_id)) data = symmetric_decrypt(resp.content, self.get_depot_key(app_id, depot_id)) if data[:2] == b'VZ': if data[-2:] != b'zv': raise SteamError("VZ: Invalid footer: %s" % repr(data[-2:])) if data[2:3] != b'a': raise SteamError("VZ: Invalid version: %s" % repr(data[2:3])) vzfilter = lzma._decode_filter_properties( lzma.FILTER_LZMA1, data[7:12]) vzdec = lzma.LZMADecompressor(lzma.FORMAT_RAW, filters=[vzfilter]) checksum, decompressed_size = struct.unpack( '<II', data[-10:-2]) # decompress_size is needed since lzma will sometime produce longer output # [12:-9] is need as sometimes lzma will produce shorter output # together they get us the right data data = vzdec.decompress(data[12:-9])[:decompressed_size] if crc32(data) != checksum: raise SteamError( "VZ: CRC32 checksum doesn't match for decompressed data" ) else: with ZipFile(BytesIO(data)) as zf: data = zf.read(zf.filelist[0]) self._chunk_cache[(depot_id, chunk_id)] = data return self._chunk_cache[(depot_id, chunk_id)]
def decompress_lump(reader: ByteIO): lzma_id = reader.read_fourcc() assert lzma_id == "LZMA", f"Unknown compressed header({lzma_id})" decompressed_size = reader.read_uint32() compressed_size = reader.read_uint32() filters = lzma._decode_filter_properties(lzma.FILTER_LZMA1, reader.read(5)) decompressor = lzma.LZMADecompressor(format=lzma.FORMAT_RAW, filters=[filters]) decompressed = decompressor.decompress(reader.read(compressed_size)) if len(decompressed) > decompressed_size: decompressed = decompressed[:decompressed_size] new_reader = ByteIO(decompressed) assert new_reader.size( ) == decompressed_size, 'Compressed lump size does not match expected' return new_reader
def _set_lzma_decompressor(self, coders: List[Dict[str, Any]]) -> None: filters = [] # type: List[Dict[str, Any]] for coder in coders: if coder['numinstreams'] != 1 or coder['numoutstreams'] != 1: raise UnsupportedCompressionMethodError( 'Only a simple compression method is currently supported.') filter_id = self.lzma_methods_map.get(coder['method'], None) if filter_id is None: raise UnsupportedCompressionMethodError properties = coder.get('properties', None) if properties is not None: filters[:0] = [ lzma._decode_filter_properties(filter_id, properties) ] # type: ignore else: filters[:0] = [{'id': filter_id}] self.decompressor = lzma.LZMADecompressor( format=lzma.FORMAT_RAW, filters=filters ) # type: Union[bz2.BZ2Decompressor, lzma.LZMADecompressor, AESDecompressor, CopyDecompressor] # noqa
def __init__(self, s: bytes): data = struct.unpack("<4s I I 5s", s[:17]) dic_size = struct.unpack("<I", s[12 + 1:12 + 1 + 4]) dic_size = dic_size[0] d = struct.unpack("B", s[12:12 + 1]) d = d[0] lc = d % 9 d = d // 9 pb = d // 5 lp = d % 5 filter_1 = lzma._decode_filter_properties(lzma.FILTER_LZMA1, data[3]) filter_2 = { "id": lzma.FILTER_LZMA1, "pb": pb, "lp": lp, "lc": lc, "dict_size": dic_size } data = lzma.decompress(s[17:], lzma.FORMAT_RAW, filters=[filter_2]) a = 1
def _lzma(f, size): import lzma data = f.read() props = lzma._decode_filter_properties(lzma.FILTER_LZMA1, data[0:5]) return lzma.decompress(data[5:], lzma.FORMAT_RAW, filters=[props])
def _init(self): props = lzma._encode_filter_properties({'id': lzma.FILTER_LZMA1}) self._comp = lzma.LZMACompressor(lzma.FORMAT_RAW, filters=[lzma._decode_filter_properties(lzma.FILTER_LZMA1, props)]) return struct.pack('<BBH', 9, 4, len(props)) + props
def update_event(self, inp=-1): self.set_output_val(0, lzma._decode_filter_properties(self.input(0), self.input(1)))
parser.add_argument( "-rawlzma1", action='store_true', help='Use Raw LZMA version 1 (default: XZ compression mode)') parser.add_argument("-rawarm", action='store_true', help='Use Raw LZMA ARM (default: XZ compression mode)') args = parser.parse_args() binary_object1 = open(args.input, 'rb').read() if args.lzma1 is True: props = lzma._encode_filter_properties({'id': lzma.FILTER_LZMA1}) lzma_comp = lzma.LZMACompressor( lzma.FORMAT_ALONE, filters=[lzma._decode_filter_properties(lzma.FILTER_LZMA1, props)], preset=args.preset) binary_object2 = lzma_comp.compress(binary_object1) lzma_comp.flush() elif args.lzma2 is True: props = lzma._encode_filter_properties({'id': lzma.FILTER_LZMA2}) lzma_comp = lzma.LZMACompressor( lzma.FORMAT_RAW, filters=[lzma._decode_filter_properties(lzma.FILTER_LZMA2, props)], preset=args.preset) binary_object2 = lzma_comp.compress(binary_object1) lzma_comp.flush() elif args.rawlzma1 is True: props = lzma._encode_filter_properties({'id': lzma.FILTER_LZMA1}) lzma_comp = lzma.LZMACompressor( lzma.FORMAT_RAW,