def unpack_value(self, reader: BinaryReader) -> PropertyValueTyping: if self in {self.Bytes, self.String}: raw_data_size = reader.unpack_value("i") value = reader.read(raw_data_size) if self == PropertyType.String: # Assumed encoding (could also be "ascii"). return value.decode("utf-8") # str return value # bytes if "Array" not in self.name: return reader.unpack_value(self.get_fmt()) array_length, is_compressed, compressed_size = reader.unpack("III") if is_compressed: decompressed_size = self.get_size(array=True) * array_length decompressed = zlib.decompressobj().decompress(reader.read(compressed_size)) if len(decompressed) != decompressed_size: raise ValueError( f"FBX property decompressed data size ({len(decompressed)}) does not match expected size " f"({decompressed_size})" ) array_reader = BinaryReader(decompressed) else: array_reader = reader fmt = self.get_fmt(array_length) return list(array_reader.unpack(fmt))
def unpack(self, esd_reader: BinaryReader, **kwargs): header = esd_reader.unpack_struct(self.EXTERNAL_HEADER_STRUCT) # Internal offsets start here, so we reset the buffer. esd_reader = BinaryReader(esd_reader.read()) internal_header = esd_reader.unpack_struct(self.INTERNAL_HEADER_STRUCT) self.magic = internal_header["magic"] state_machine_headers = esd_reader.unpack_structs( self.STATE_MACHINE_HEADER_STRUCT, count=header["state_machine_count"]) for state_machine_header in state_machine_headers: states = self.State.unpack( esd_reader, state_machine_header["state_machine_offset"], count=state_machine_header["state_count"], ) self.state_machines[ state_machine_header["state_machine_index"]] = states if internal_header["esd_name_length"] > 0: esd_name_offset = internal_header["esd_name_offset"] esd_name_length = internal_header["esd_name_length"] # Note the given length is the length of the final string. The actual UTF-16 encoded bytes are twice that. self.esd_name = esd_reader.unpack_string(offset=esd_name_offset, length=2 * esd_name_length, encoding="utf-16le") esd_reader.seek(esd_name_offset + 2 * esd_name_length) self.file_tail = esd_reader.read() else: self.esd_name = "" esd_reader.seek(header["unk_offset_1"]) # after packed EZL self.file_tail = esd_reader.read()
def unpack_from(cls, reader: BinaryReader, data_offset: int): header = FSBSampleHeader(reader) metadata_size = header.total_size - FSBSampleHeader.STRUCT.size metadata = reader.read(metadata_size) with reader.temp_offset(data_offset): data = reader.read(header.compressed_length) return FSBSample(header, metadata, data)
def unpack(self, emevd_reader: BinaryReader, **kwargs): header = emevd_reader.unpack_struct(self.HEADER_STRUCT) emevd_reader.seek(header["event_table_offset"]) event_dict = self.Event.unpack_event_dict( emevd_reader, header["instruction_table_offset"], header["base_arg_data_offset"], header["event_arg_table_offset"], header["event_layers_table_offset"], count=header["event_count"], ) self.events.update(event_dict) if header["packed_strings_size"] != 0: emevd_reader.seek(header["packed_strings_offset"]) self.packed_strings = emevd_reader.read( header["packed_strings_size"]) if header["linked_files_count"] != 0: emevd_reader.seek(header["linked_files_table_offset"]) # These are relative offsets into the packed string data. for _ in range(header["linked_files_count"]): self.linked_file_offsets.append( struct.unpack("<Q", emevd_reader.read(8))[0]) # Parse event args for `RunEvent` and `RunCommonEvent` instructions. for event in self.events.values(): event.update_evs_function_args() for event in self.events.values(): event.update_run_event_instructions()
def unpack(self, row_reader: BinaryReader): for field in self.paramdef.fields.values(): if field.bit_count != -1: field_value = self.bit_reader.read(row_reader, field.bit_count, field.fmt) else: self.bit_reader.clear() if issubclass(field.type_class, ft.basestring): field_value = field.type_class.read(row_reader, field.size) elif field.type_class is ft.dummy8: # These are often 'array' fields, but we don't even bother unpacking them. field_value = row_reader.read(field.size) else: data = row_reader.read(field.type_class.size()) try: field_value = struct.unpack(field.fmt, data)[0] except struct.error as e: if field.display_name in { "inverseToneMapMul", "sfxMultiplier" }: # These fields are malformed in m99 and default ToneMapBank in Dark Souls Remastered. field_value = 1.0 else: raise ValueError( f"Could not unpack data for field {field.name} in ParamRow {self.name}.\n" f"Field type: {field.display_type}\n" f"Raw bytes: {data}\n" f"Error:\n{str(e)}") self.fields[field.name] = bool( field_value) if field.bit_count == 1 else field_value
def decrypt_regulation_bin(self, data: bytes) -> bytes: try: # noinspection PyPackageRequirements from aespython import key_expander, aes_cipher, cbc_mode except ImportError: raise ModuleNotFoundError( f"Cannot decrypt `regulation.bin` for Elden Ring without `aespython` package." ) iv = data[:16] encrypted = BinaryReader(data) key_expander_256 = key_expander.KeyExpander(256) expanded_key = key_expander_256.expand(bytearray(self.REGULATION_KEY)) aes_cipher_256 = aes_cipher.AESCipher(expanded_key) aes_cbc_256 = cbc_mode.CBCMode(aes_cipher_256, 16) aes_cbc_256.set_iv(iv) decrypted = b"" while True: chunk = encrypted.read(16) if len(chunk) == 0: # end of file break decrypted_chunk = bytearray( aes_cbc_256.decrypt_block(list(bytearray(chunk)))) decrypted += decrypted_chunk # print(f"{len(decrypted)}") with open("regulation.bin.decrypted", "wb") as f: f.write(decrypted) with open("regulation.bin.decryptednodcx", "wb") as f: f.write(decompress(decrypted[16:])[0]) return decrypted
def unpack(self, reader: BinaryReader, **kwargs): entry_count, entry_header_size, hash_table_offset, data_offset = self.unpack_header( reader) flags_header_size = self.flags.get_bnd_entry_header_size() if entry_header_size != flags_header_size: raise ValueError( f"Expected BND entry header size {flags_header_size} based on flags\n" f"{self.flags:08b}, but BND header says {entry_header_size}.") if self.hash_table_type != 4 and hash_table_offset != 0: _LOGGER.warning( f"Found non-zero hash table offset {hash_table_offset}, but header says this BND has no hash " f"table.") entry_headers = [ BinderEntryHeader.from_bnd4_reader(reader, self.flags, self.bit_big_endian, self.unicode) for _ in range(entry_count) ] for entry_header in entry_headers: self.add_entry(BinderEntry.from_header(reader, entry_header)) if self.hash_table_type == 4: # Save the initial hash table. reader.seek(hash_table_offset) self._most_recent_hash_table = reader.read(data_offset - hash_table_offset) self._most_recent_entry_count = len(self._entries) self._most_recent_paths = [entry.path for entry in self._entries]
def decompress(dcx_source: ReadableTyping) -> tuple[bytes, DCXType]: """Decompress the given file path, raw bytes, or buffer/reader. Returns a tuple containing the decompressed `bytes` and a `DCXInfo` instance that can be used to compress later with the same DCX type/parameters. """ reader = BinaryReader(dcx_source, byte_order=">") # always big-endian dcx_type = DCXType.detect(reader) if dcx_type == DCXType.Unknown: raise ValueError("Unknown DCX type. Cannot decompress.") header = reader.unpack_struct(DCX_HEADER_STRUCTS[dcx_type], byte_order=">") compressed = reader.read( header["compressed_size"]) # TODO: do I need to rstrip nulls? if dcx_type == DCXType.DCX_KRAK: decompressed = oodle.decompress(compressed, header["decompressed_size"]) else: decompressed = zlib.decompressobj().decompress(compressed) if len(decompressed) != header["decompressed_size"]: raise ValueError( "Decompressed DCX data size does not match size in header.") return decompressed, dcx_type
def unpack(self, reader: BinaryReader): self.gx_items = [] while reader.unpack_value("<i", offset=reader.position) not in {2 ** 31 - 1, -1}: self.gx_items.append(GXItem(reader)) self.terminator_id = reader.unpack_value("<i") # either 2 ** 31 - 1 or -1 reader.unpack_value("<i", asserted=100) self.terminator_null_count = reader.unpack_value("<i") - 12 terminator_nulls = reader.read(self.terminator_null_count) if terminator_nulls.strip(b"\0"): raise ValueError(f"Found non-null data in terminator: {terminator_nulls}")
def unpack(self, reader: BinaryReader, **kwargs): self.big_endian, self.use_struct_64 = self._check_big_endian_and_struct_64( reader) fmt = f"{'>' if self.big_endian else '<'}{'q' if self.use_struct_64 else 'i'}" read_size = struct.calcsize(fmt) self.names = [] offset = None while offset != 0: (offset, ) = struct.unpack(fmt, reader.read(read_size)) if offset != 0: self.names.append( reader.unpack_string(offset=offset, encoding=self.encoding))
def unpack_header(self, reader: BinaryReader): reader.unpack_value("4s", asserted=b"BND4") self.unknown1 = reader.unpack_value("?") self.unknown2 = reader.unpack_value("?") reader.assert_pad(3) self.big_endian = reader.unpack_value("?") self.bit_big_endian = not reader.unpack_value("?") # note reversal reader.assert_pad(1) reader.byte_order = ">" if self.big_endian else "<" # no need to check flags for an override in BND4 entry_count = reader.unpack_value("i") reader.unpack_value("q", asserted=0x40) # header size self.signature = reader.unpack_value("8s").decode("ascii").rstrip("\0") entry_header_size = reader.unpack_value("q") data_offset = reader.unpack_value( "q") # end of all headers, including hash table self.unicode = reader.unpack_value("?") self.flags = BinderFlags.read(reader, self.bit_big_endian) self.hash_table_type = reader.unpack_value("B") reader.assert_pad(5) hash_table_offset = reader.unpack_value("q") flags_header_size = self.flags.get_bnd_entry_header_size() if entry_header_size != flags_header_size: raise ValueError( f"Expected BND entry header size {flags_header_size} based on flags\n" f"{self.flags:08b}, but BND header says {entry_header_size}.") if self.hash_table_type != 4 and hash_table_offset != 0: _LOGGER.warning( f"Found non-zero hash table offset {hash_table_offset}, but header says this BHD has no hash " f"table.") entry_headers = [ BinderEntryHeader.from_bnd4_reader(reader, self.flags, self.bit_big_endian, self.unicode) for _ in range(entry_count) ] if self.hash_table_type == 4: # Save the initial hash table. reader.seek(hash_table_offset) self._most_recent_hash_table = reader.read(data_offset - hash_table_offset) return entry_headers
def unpack(cls, reader: BinaryReader, base_arg_data_offset, event_layers_table_offset, count=1): """Unpack some number of Instructions into a list, starting from the current file offset.""" instructions = [] struct_dicts = reader.unpack_structs(cls.HEADER_STRUCT, count=count) for i, d in enumerate(struct_dicts): # Process arguments. try: args_format, args_list = get_instruction_args( reader, d["category"], d["index"], base_arg_data_offset + d["first_base_arg_offset"], d["base_args_size"], cls.INSTRUCTION_ARG_TYPES, ) except KeyError: args_size = struct_dicts[ i + 1]["first_base_arg_offset"] - d["first_base_arg_offset"] reader.seek(base_arg_data_offset + d["first_base_arg_offset"]) raw_data = reader.read(args_size) _LOGGER.error( f"Error while processing instruction arguments. Raw arg data: {raw_data}" ) raise # Process event layers. if d["first_event_layers_offset"] > 0: event_layers = cls.EventLayers.unpack( reader, event_layers_table_offset + d["first_event_layers_offset"]) else: event_layers = None instructions.append( cls(d["category"], d["index"], args_format, args_list, event_layers)) return instructions
def unpack(self, dcx_reader: BinaryReader): if self.magic: raise ValueError( "`DCX.magic` cannot be set manually before unpack.") header = dcx_reader.unpack_struct(self.HEADER_STRUCT) self.magic = header["magic"] compressed = dcx_reader.read().rstrip( b"\0") # Nulls stripped from the end. if len(compressed) != header["compressed_size"]: # No error raised. This happens in some files. file_path = f" {self.dcx_path}" if self.dcx_path else "" _LOGGER.warning( f"Compressed data size ({len(compressed)}) does not match size in header " f"({header['compressed_size']}) in DCX-compressed file{file_path}." ) self.data = zlib.decompressobj().decompress(compressed) if len(self.data) != header["decompressed_size"]: raise ValueError( "Decompressed data size does not match size in header.")
def check_null_field(cls, msb_reader: BinaryReader, offset_to_null): msb_reader.seek(offset_to_null) zero = msb_reader.read(cls.UNKNOWN_DATA_SIZE) if zero != b"\0" * cls.UNKNOWN_DATA_SIZE: _LOGGER.warning( f"Null data entry in `{cls.__name__}` was not zero: {zero}.")
def from_header(cls, binder_reader: BinaryReader, entry_header: BinderEntryHeader) -> BinderEntry: with binder_reader.temp_offset(entry_header.data_offset): data = binder_reader.read(entry_header.compressed_size) return cls(entry_id=entry_header.id, path=entry_header.path, data=data, flags=entry_header.flags)
def unpack_from( cls, reader: BinaryReader, platform: TPFPlatform, tpf_flags: int, encoding: str, tpf_path: tp.Union[None, str, Path] = None, ): self = cls() self.tpf_path = tpf_path file_offset = reader.unpack_value("I") file_size = reader.unpack_value("i") self.format = reader.unpack_value("B") self.texture_type = TextureType(reader.unpack_value("B")) self.mipmaps = reader.unpack_value("B") self.texture_flags = reader.unpack_value("B") if self.texture_flags not in {0, 1, 2, 3}: raise ValueError( f"`TPFTexture.flags1` was {self.texture_flags}, but expected 0, 1, 2, or 3." ) if platform != TPFPlatform.PC: self.header = TextureHeader self.header.width = reader.unpack_value("h") self.header.height = reader.unpack_value("h") if platform == TPFPlatform.Xbox360: reader.assert_pad(4) elif platform == TPFPlatform.PS3: self.header.unk1 = reader.unpack_value("i") if tpf_flags != 0: self.header.unk2 = reader.unpack_value("i") if self.header.unk2 not in {0, 0x68E0, 0xAAE4}: raise ValueError( f"`TextureHeader.unk2` was {self.header.unk2}, but expected 0, 0x68E0, or 0xAAE4." ) elif platform in {TPFPlatform.PS4, TPFPlatform.XboxOne}: self.header.texture_count = reader.unpack_value("i") if self.header.texture_count not in {1, 6}: f"`TextureHeader.texture_count` was {self.header.texture_count}, but expected 1 or 6." self.header.unk2 = reader.unpack_value("i") if self.header.unk2 != 0xD: f"`TextureHeader.unk2` was {self.header.unk2}, but expected 0xD." name_offset = reader.unpack_value("I") has_float_struct = reader.unpack_value("i") == 1 if platform in {TPFPlatform.PS4, TPFPlatform.XboxOne}: self.header.dxgi_format = reader.unpack_value("i") if has_float_struct: self.float_struct = FloatStruct.unpack_from(reader) with reader.temp_offset(file_offset): self.data = reader.read(file_size) if self.texture_flags in {2, 3}: # Data is DCX-compressed. # TODO: should enforce DCX type as 'DCP_EDGE'? self.data = decompress(self.data) self.name = reader.unpack_string(offset=name_offset, encoding=encoding) return self
def unpack(self, reader: BinaryReader): gx_item = reader.unpack_struct(self.STRUCT) self.data = reader.read(gx_item.pop("__size") - self.STRUCT.size) self.set(**gx_item)
def unpack(self, reader: BinaryReader, **kwargs): self.byte_order = reader.byte_order = ">" if reader.unpack_value( "B", offset=44) == 255 else "<" version_info = reader.unpack("bbb", offset=45) self.flags1 = ParamFlags1(version_info[0]) self.flags2 = ParamFlags2(version_info[1]) self.paramdef_format_version = version_info[2] header_struct = self.GET_HEADER_STRUCT(self.flags1, self.byte_order) header = reader.unpack_struct(header_struct) try: self.param_type = header["param_type"] except KeyError: self.param_type = reader.unpack_string( offset=header["param_type_offset"], encoding="utf-8") self.paramdef_data_version = header["paramdef_data_version"] self.unknown = header["unknown"] # Row data offset in header not used. (It's an unsigned short, yet doesn't limit row count to 5461.) name_data_offset = header[ "name_data_offset"] # CANNOT BE TRUSTED IN VANILLA FILES! Off by +12 bytes. # Load row pointer data. row_struct = self.ROW_STRUCT_64 if self.flags1.LongDataOffset else self.ROW_STRUCT_32 row_pointers = reader.unpack_structs(row_struct, count=header["row_count"]) row_data_offset = reader.position # Reliable row data offset. # Row size is lazily determined. TODO: Unpack row data in sequence and associate with names separately. if len(row_pointers) == 0: return elif len(row_pointers) == 1: # NOTE: The only vanilla param in Dark Souls with one row is LEVELSYNC_PARAM_ST (Remastered only), # for which the row size is hard-coded here. Otherwise, we can trust the repacked offset from Soulstruct # (and SoulsFormats, etc.). if self.param_type == "LEVELSYNC_PARAM_ST": row_size = 220 else: row_size = name_data_offset - row_data_offset else: row_size = row_pointers[1]["data_offset"] - row_pointers[0][ "data_offset"] # Note that we no longer need to track reader offset. name_encoding = self.get_name_encoding() for row_struct in row_pointers: reader.seek(row_struct["data_offset"]) row_data = reader.read(row_size) if row_struct["name_offset"] != 0: try: name = reader.unpack_string( offset=row_struct["name_offset"], encoding=name_encoding, reset_old_offset=False, # no need to reset ) except UnicodeDecodeError as ex: if ex.object in self.undecodable_row_names: name = reader.unpack_bytes( offset=row_struct["name_offset"], reset_old_offset=False, # no need to reset ) else: raise except ValueError: reader.seek(row_struct["name_offset"]) _LOGGER.error( f"Error encountered while parsing row name string in {self.param_type}.\n" f" Header: {header}\n" f" Row Struct: {row_struct}\n" f" 30 chrs of name data: {' '.join(f'{{:02x}}'.format(x) for x in reader.read(30))}" ) raise else: name = "" self.rows[row_struct["id"]] = ParamRow(row_data, self.paramdef, name=name)
def read(self, reader: BinaryReader, layout: BufferLayout, uv_factor: float): self.uvs = [] self.tangents = [] self.colors = [] with reader.temp_offset(reader.position): self.raw = reader.read(layout.get_total_size()) for member in layout: not_implemented = False if member.semantic == LayoutSemantic.Position: if member.layout_type == LayoutType.Float3: self.position = Vector3(reader.unpack("<3f")) elif member.layout_type == LayoutType.Float4: self.position = Vector3(reader.unpack("<3f"))[:3] elif member.layout_type == LayoutType.EdgeCompressed: raise NotImplementedError( "Soulstruct cannot load FLVERs with edge-compressed vertex positions." ) else: not_implemented = True elif member.semantic == LayoutSemantic.BoneWeights: if member.layout_type == LayoutType.Byte4A: self.bone_weights = VertexBoneWeights( *[w / 127.0 for w in reader.unpack("<4b")]) elif member.layout_type == LayoutType.Byte4C: self.bone_weights = VertexBoneWeights( *[w / 255.0 for w in reader.unpack("<4B")]) elif member.layout_type in { LayoutType.UVPair, LayoutType.Short4ToFloat4A }: self.bone_weights = VertexBoneWeights( *[w / 32767.0 for w in reader.unpack("<4h")]) else: not_implemented = True elif member.semantic == LayoutSemantic.BoneIndices: if member.layout_type in { LayoutType.Byte4B, LayoutType.Byte4E }: self.bone_indices = VertexBoneIndices( *reader.unpack("<4B")) elif member.layout_type == LayoutType.ShortBoneIndices: self.bone_indices = VertexBoneIndices( *reader.unpack("<4h")) else: not_implemented = True elif member.semantic == LayoutSemantic.Normal: if member.layout_type == LayoutType.Float3: self.normal = Vector3(reader.unpack("<3f")) elif member.layout_type == LayoutType.Float4: self.normal = Vector3(reader.unpack("<3f")) float_normal_w = reader.unpack_value("<f") self.normal_w = int(float_normal_w) if self.normal_w != float_normal_w: raise ValueError( f"`normal_w` float was not a whole number.") elif member.layout_type in { LayoutType.Byte4A, LayoutType.Byte4B, LayoutType.Byte4C, LayoutType.Byte4E }: self.normal = Vector3([(x - 127) / 127.0 for x in reader.unpack("<3B")]) self.normal_w = reader.unpack_value("<B") elif member.layout_type == LayoutType.Short2toFloat2: self.normal_w = reader.unpack_value("<B") self.normal = Vector3( [x / 127.0 for x in reader.unpack("<3b")]) elif member.layout_type == LayoutType.Short4ToFloat4A: self.normal = Vector3( [x / 32767.0 for x in reader.unpack("<3h")]) self.normal_w = reader.unpack_value("<h") elif member.layout_type == LayoutType.Short4ToFloat4B: self.normal = Vector3([(x - 32767) / 32767.0 for x in reader.unpack("<3H")]) self.normal_w = reader.unpack_value("<h") else: not_implemented = True elif member.semantic == LayoutSemantic.UV: if member.layout_type == LayoutType.Float2: self.uvs.append(Vector3(*reader.unpack("<2f"), 0.0)) elif member.layout_type == LayoutType.Float3: self.uvs.append(Vector3(*reader.unpack("<3f"))) elif member.layout_type == LayoutType.Float4: self.uvs.append(Vector3(*reader.unpack("<2f"), 0.0)) self.uvs.append(Vector3(*reader.unpack("<2f"), 0.0)) elif member.layout_type in { LayoutType.Byte4A, LayoutType.Byte4B, LayoutType.Short2toFloat2, LayoutType.Byte4C, LayoutType.UV }: self.uvs.append( Vector3(*reader.unpack("<2h"), 0) / uv_factor) elif member.layout_type == LayoutType.UVPair: self.uvs.append( Vector3(*reader.unpack("<2h"), 0) / uv_factor) self.uvs.append( Vector3(*reader.unpack("<2h"), 0) / uv_factor) elif member.layout_type == LayoutType.Short4ToFloat4B: self.uvs.append(Vector3(*reader.unpack("<3h")) / uv_factor) if reader.unpack_value("<h") != 0: raise ValueError( "Expected zero short after reading UV | Short4ToFloat4B vertex member." ) else: not_implemented = True elif member.semantic == LayoutSemantic.Tangent: if member.layout_type == LayoutType.Float4: self.tangents.append(Vector4(*reader.unpack("<4f"))) elif member.layout_type in { LayoutType.Byte4A, LayoutType.Byte4B, LayoutType.Byte4C, LayoutType.Short4ToFloat4A, LayoutType.Byte4E, }: tangent = Vector4([(x - 127) / 127.0 for x in reader.unpack("<4B")]) self.tangents.append(tangent) else: not_implemented = True elif member.semantic == LayoutSemantic.Bitangent: if member.layout_type in { LayoutType.Byte4A, LayoutType.Byte4B, LayoutType.Byte4C, LayoutType.Byte4E }: self.bitangent = Vector4([(x - 127) / 127.0 for x in reader.unpack("<4B")]) else: not_implemented = True elif member.semantic == LayoutSemantic.VertexColor: if member.layout_type == LayoutType.Float4: self.colors.append(ColorRGBA(*reader.unpack("<4f"))) elif member.layout_type in { LayoutType.Byte4A, LayoutType.Byte4C }: # Convert byte channnels [0-255] to float channels [0-1]. self.colors.append( ColorRGBA(*[b / 255.0 for b in reader.unpack("<4B")])) else: not_implemented = True else: not_implemented = True if not_implemented: raise NotImplementedError( f"Unsupported vertex member semantic/type combination: " f"{member.semantic.name} | {member.layout_type.name}")
def _is_dcx(reader: BinaryReader) -> bool: """Checks if file data starts with "DCX" magic.""" with reader.temp_offset(offset=0): return reader.read(4) == b"DCX\0"