def unpack(self, msb_reader: BinaryReader): part_offset = msb_reader.position header = msb_reader.unpack_struct(self.PART_HEADER_STRUCT) if header["__part_type"] != self.ENTRY_SUBTYPE: raise ValueError(f"Unexpected part type enum {header['part_type']} for class {self.__class__.__name__}.") self._instance_index = header["_instance_index"] self._model_index = header["_model_index"] self._part_type_index = header["_part_type_index"] for transform in ("translate", "rotate", "scale"): setattr(self, transform, Vector3(header[transform])) self._draw_groups = int_group_to_bit_set(header["__draw_groups"], assert_size=8) self._display_groups = int_group_to_bit_set(header["__display_groups"], assert_size=8) self._backread_groups = int_group_to_bit_set(header["__backread_groups"], assert_size=8) self.description = msb_reader.unpack_string( offset=part_offset + header["__description_offset"], encoding="utf-16-le", ) self.name = msb_reader.unpack_string( offset=part_offset + header["__name_offset"], encoding="utf-16-le", ) self.sib_path = msb_reader.unpack_string( offset=part_offset + header["__sib_path_offset"], encoding="utf-16-le", ) msb_reader.seek(part_offset + header["__base_data_offset"]) base_data = msb_reader.unpack_struct(self.PART_BASE_DATA_STRUCT) self.set(**base_data) msb_reader.seek(part_offset + header["__type_data_offset"]) self.unpack_type_data(msb_reader) self._unpack_gparam_data(msb_reader, part_offset, header) self._unpack_scene_gparam_data(msb_reader, part_offset, header)
def unpack(self, msb_reader: BinaryReader): model_offset = msb_reader.position model_data = msb_reader.unpack_struct(self.MODEL_STRUCT) self.name = msb_reader.unpack_string( offset=model_offset + model_data["__name_offset"], encoding=self.NAME_ENCODING ) self.sib_path = msb_reader.unpack_string( offset=model_offset + model_data["__sib_path_offset"], encoding=self.NAME_ENCODING, ) try: self.ENTRY_SUBTYPE = MSBModelSubtype(model_data["__model_type"]) except TypeError: raise ValueError(f"Unrecognized MSB model type: {model_data['__model_type']}") self.set(**model_data)
def unpack(self, msb_reader: BinaryReader): model_offset = msb_reader.position header = msb_reader.unpack_struct(self.MODEL_STRUCT) self.name = msb_reader.unpack_string(offset=model_offset + header["__name_offset"], encoding=self.NAME_ENCODING) self.sib_path = msb_reader.unpack_string( offset=model_offset + header["__sib_path_offset"], encoding=self.NAME_ENCODING, ) if header["__model_type"] != self.ENTRY_SUBTYPE.value: raise ValueError( f"Unexpected MSB model type value {header['__model_type']} for {self.__class__.__name__}. " f"Expected {self.ENTRY_SUBTYPE.value}.") self.set(**header)
def unpack(self, reader: BinaryReader, remove_empty_entries=True): header = reader.unpack_struct(self.HEADER_STRUCT) # Groups of contiguous text string IDs are defined by ranges (first ID, last ID) to save space. ranges = reader.unpack_structs(self.RANGE_STRUCT, count=header["range_count"]) if reader.position != header["string_offsets_offset"]: _LOGGER.warning( "Range data did not end at string data offset given in FMG header." ) string_offsets = reader.unpack_structs(self.STRING_OFFSET_STRUCT, count=header["string_count"]) # Text pointer table corresponds to all the IDs (joined together) of the above ranges, in order. for string_range in ranges: i = string_range["first_index"] for string_id in range(string_range["first_id"], string_range["last_id"] + 1): if string_id in self.entries: raise ValueError( f"Malformed FMG: Entry index {string_id} appeared more than once." ) string_offset = string_offsets[i]["offset"] if string_offset == 0: if not remove_empty_entries: # Empty text string. These will trigger in-game error messages, like ?PlaceName?. # Distinct from ' ', which is intentionally blank text data (e.g. the unused area subtitles). self.entries[string_id] = "" else: string = reader.unpack_string(offset=string_offset, encoding="utf-16le") if string or not remove_empty_entries: self.entries[string_id] = string i += 1
def unpack(self, reader: BinaryReader, start_offset: int, depth=0): data = reader.unpack_struct(self.STRUCT) name_length = data.pop("__name_length") self.name = reader.unpack_string(length=name_length, encoding="ascii") self.size = self.STRUCT.size + name_length self.depth = depth # TODO: Use `_properties` and `field` properties, which inspect the node name, etc. self._properties = [FBXProperty.unpack(reader) for _ in range(data.pop("__property_count"))] self.size += data.pop("__property_list_size") self.children = [] end_offset = data.pop("__end_offset") while start_offset + self.size < end_offset: child = self.__class__(reader, start_offset=start_offset + self.size, depth=self.depth + 1) self.size += child.size if start_offset + self.size == end_offset: break # empty node is not kept self.children.append(child) if self.name == "P": if self.children: raise ValueError("`FBXNode` named 'P' should not have any children.") name, *args = [p.value for p in self._properties] self._field = FBXPropertyField(name, *args) else: self._field = None
def unpack(self, esd_reader: BinaryReader, **kwargs): header = esd_reader.unpack_struct(self.EXTERNAL_HEADER_STRUCT) # Internal offsets start here, so we reset the buffer. esd_reader = BinaryReader(esd_reader.read()) internal_header = esd_reader.unpack_struct(self.INTERNAL_HEADER_STRUCT) self.magic = internal_header["magic"] state_machine_headers = esd_reader.unpack_structs( self.STATE_MACHINE_HEADER_STRUCT, count=header["state_machine_count"]) for state_machine_header in state_machine_headers: states = self.State.unpack( esd_reader, state_machine_header["state_machine_offset"], count=state_machine_header["state_count"], ) self.state_machines[ state_machine_header["state_machine_index"]] = states if internal_header["esd_name_length"] > 0: esd_name_offset = internal_header["esd_name_offset"] esd_name_length = internal_header["esd_name_length"] # Note the given length is the length of the final string. The actual UTF-16 encoded bytes are twice that. self.esd_name = esd_reader.unpack_string(offset=esd_name_offset, length=2 * esd_name_length, encoding="utf-16le") esd_reader.seek(esd_name_offset + 2 * esd_name_length) self.file_tail = esd_reader.read() else: self.esd_name = "" esd_reader.seek(header["unk_offset_1"]) # after packed EZL self.file_tail = esd_reader.read()
def unpack_strings(self) -> list[tuple[str, str]]: strings = [] string_reader = BinaryReader(self.packed_strings) while string_reader.position != len(self.packed_strings): offset = string_reader.position string = string_reader.unpack_string(encoding=self.STRING_ENCODING) strings.append((str(offset), string)) return strings
def unpack_goal(self, reader: BinaryReader, goal_struct: BinaryStruct) -> LuaGoal: goal = reader.unpack_struct(goal_struct, byte_order=">" if self.big_endian else "<") name = reader.unpack_string(offset=goal["name_offset"], encoding=self.encoding) if goal["logic_interrupt_name_offset"] > 0: logic_interrupt_name = reader.unpack_string( offset=goal["logic_interrupt_name_offset"], encoding=self.encoding) else: logic_interrupt_name = "" return LuaGoal( goal_id=goal["goal_id"], goal_name=name, has_battle_interrupt=goal["has_battle_interrupt"], has_logic_interrupt=goal["has_logic_interrupt"], logic_interrupt_name=logic_interrupt_name, )
def unpack( self, reader: BinaryReader, encoding: str, version: Version, gx_lists: tp.List[GXList], gx_list_indices: tp.Dict[int, int], ): material = reader.unpack_struct(self.STRUCT) self.name = reader.unpack_string(offset=material.pop("__name__z"), encoding=encoding) self.mtd_path = reader.unpack_string(offset=material.pop("__mtd_path__z"), encoding=encoding) gx_offset = material.pop("__gx_offset") if gx_offset == 0: self.gx_index = -1 elif gx_offset in gx_list_indices: self.gx_index = gx_list_indices[gx_offset] else: self.gx_index = gx_list_indices[gx_offset] = len(gx_lists) with reader.temp_offset(gx_offset): gx_lists.append(GXList(reader, version)) self.set(**material)
def unpack(self, reader: BinaryReader, **kwargs): self.big_endian, self.use_struct_64 = self._check_big_endian_and_struct_64( reader) fmt = f"{'>' if self.big_endian else '<'}{'q' if self.use_struct_64 else 'i'}" read_size = struct.calcsize(fmt) self.names = [] offset = None while offset != 0: (offset, ) = struct.unpack(fmt, reader.read(read_size)) if offset != 0: self.names.append( reader.unpack_string(offset=offset, encoding=self.encoding))
def unpack(self, msb_reader: BinaryReader): event_offset = msb_reader.position header = msb_reader.unpack_struct(self.EVENT_HEADER_STRUCT) if header["__event_type"] != self.ENTRY_SUBTYPE: raise ValueError(f"Unexpected MSB event type value {header['__event_type']} for {self.__class__.__name__}.") msb_reader.seek(event_offset + header["__base_data_offset"]) base_data = msb_reader.unpack_struct(self.EVENT_BASE_DATA_STRUCT) name_offset = event_offset + header["__name_offset"] self.name = msb_reader.unpack_string(offset=name_offset, encoding=self.NAME_ENCODING) self.set(**header) self.set(**base_data) msb_reader.seek(event_offset + header["__type_data_offset"]) self.unpack_type_data(msb_reader)
def unpack_fields( cls, param_name: str, paramdef_reader: BinaryReader, field_count: int, format_version: int, unicode: bool, byte_order: str, ) -> dict[str, ParamDefField]: """Buffer should be at the start of the packed fields (which are followed by the packed descriptions).""" field_structs = paramdef_reader.unpack_structs(cls.GET_FIELD_STRUCT( format_version, unicode, byte_order), count=field_count) fields = {} for field_index, field_struct in enumerate(field_structs): if field_struct["description_offset"] != 0: field_description = paramdef_reader.unpack_string( offset=field_struct["description_offset"], encoding="utf-16-le" if unicode else "shift_jis_2004", ) else: field_description = "" if "display_name_offset" in field_struct: display_name = paramdef_reader.unpack_string( offset=field_struct["display_name_offset"], encoding="utf-16-le", ) else: display_name = field_struct["display_name"] field = cls(field_struct, field_index, field_description, param_name, display_name=display_name) fields[field.name] = field return fields
def detect(cls, binder_source: tp.Union[GameFile.Typing, dict]) -> bool: """Returns True if `binder_source` appears to be this subclass of `BaseBinder`. Does not support DCX sources.""" if isinstance(binder_source, dict): # Manifest dictionary. Simply check version. return binder_source.get( "version") == cls.__name__ # "BND3", "BND4", etc. if isinstance(binder_source, (str, Path)): binder_path = Path(binder_source) if binder_path.is_file( ) and binder_path.name == "binder_manifest.json": binder_path = binder_path.parent if binder_path.is_dir(): try: manifest = read_json(binder_path / "binder_manifest.json", encoding="shift_jis") return manifest.get( "version") == cls.__name__ # "BND3", "BND4", etc. except FileNotFoundError: return False elif binder_path.is_file(): reader = BinaryReader(binder_path) try: version = reader.unpack_string(length=4, encoding="ascii") except ValueError: return False if version[:3] in {"BHF", "BDF"}: version = f"BXF{version[3]}" # BXF header or data file return version == cls.__name__ return False elif isinstance(binder_source, (bytes, io.BufferedIOBase)): binder_source = BinaryReader(binder_source) if isinstance(binder_source, BinaryReader): with binder_source.temp_offset(0): try: version = binder_source.unpack_string(length=4, encoding="ascii") except ValueError: return False if version[:3] in {"BHF", "BDF"}: version = f"BXF{version[3]}" # BXF header or data file return version == cls.__name__ raise TypeError( f"Cannot detect `Binder` class from source type: {binder_source}")
def unpack(self, paramdef_reader: BinaryReader, **kwargs): header = paramdef_reader.unpack_struct(self.HEADER_STRUCT) if "param_name" in header: self.param_type = header["param_name"] else: self.param_type = paramdef_reader.unpack_string( offset=header["param_name_offset"], encoding="shift_jis_2004", # never unicode ) self.data_version = header["data_version"] self.format_version = header["format_version"] self.unicode = header["unicode"] self.fields = self.FIELD_CLASS.unpack_fields( self.param_type, paramdef_reader, header["field_count"], self.format_version, self.unicode, self.BYTE_ORDER, )
def from_bnd3_reader(cls, reader: BinaryReader, binder_flags: BinderFlags, bit_big_endian: bool): flags = BinderEntryFlags.read(reader, bit_big_endian) reader.assert_pad(3) compressed_size = reader.unpack_value("i") data_offset = reader.unpack_value("q" if binder_flags.has_long_offsets else "I") entry_id = reader.unpack_value("i") if binder_flags.has_ids else None if binder_flags.has_names: path_offset = reader.unpack_value("i") path = reader.unpack_string(path_offset, encoding="shift-jis") # NOT `shift_jis_2004` else: path = None uncompressed_size = reader.unpack_value("i") if binder_flags.has_compression else None return cls( flags=flags, compressed_size=compressed_size, entry_id=entry_id, path=path, uncompressed_size=uncompressed_size, data_offset=data_offset, )
def from_bnd4_reader(cls, reader: BinaryReader, binder_flags: BinderFlags, bit_big_endian: bool, unicode: bool): flags = BinderEntryFlags.read(reader, bit_big_endian) reader.assert_pad(3) assert reader.unpack_value("i") == -1 compressed_size = reader.unpack_value("q") uncompressed_size = reader.unpack_value("q") if binder_flags.has_compression else None data_offset = reader.unpack_value("q" if binder_flags.has_long_offsets else "I") entry_id = reader.unpack_value("i") if binder_flags.has_ids else None if binder_flags.has_names: path_offset = reader.unpack_value("I") path = reader.unpack_string(path_offset, encoding="utf-16-le" if unicode else "shift-jis") else: path = None return cls( flags=flags, compressed_size=compressed_size, entry_id=entry_id, path=path, uncompressed_size=uncompressed_size, data_offset=data_offset, )
def unpack(self, msb_reader: BinaryReader): header = msb_reader.unpack_struct(self.MAP_ENTITY_LIST_HEADER) entry_offsets = [ msb_reader.unpack_struct( self.MAP_ENTITY_ENTRY_OFFSET)["entry_offset"] for _ in range(header["entry_offset_count"] - 1) # 'entry_offset_count' includes tail offset ] next_entry_list_offset = msb_reader.unpack_struct( self.MAP_ENTITY_LIST_TAIL)["next_entry_list_offset"] self.name = msb_reader.unpack_string(offset=header["name_offset"], encoding=self.NAME_ENCODING) self._entries = [] for entry_offset in entry_offsets: msb_reader.seek(entry_offset) entry = self.ENTRY_CLASS(msb_reader) self._entries.append(entry) msb_reader.seek(next_entry_list_offset)
def unpack(self, msb_reader: BinaryReader): region_offset = msb_reader.position base_data = msb_reader.unpack_struct(self.REGION_STRUCT) self.name = msb_reader.unpack_string( offset=region_offset + base_data["name_offset"], encoding=self.NAME_ENCODING, ) self._region_index = base_data["__region_index"] self.translate = Vector3(base_data["translate"]) self.rotate = Vector3(base_data["rotate"]) self.check_null_field(msb_reader, region_offset + base_data["unknown_offset_1"]) self.check_null_field(msb_reader, region_offset + base_data["unknown_offset_2"]) if base_data["type_data_offset"] != 0: msb_reader.seek(region_offset + base_data["type_data_offset"]) self.unpack_type_data(msb_reader) msb_reader.seek(region_offset + base_data["entity_id_offset"]) self.entity_id = msb_reader.unpack_value("i") return region_offset + base_data["entity_id_offset"]
def detect(cls, binder_source: GameFile.Typing) -> bool: """Returns True if `binder_source` appears to be this subclass of `BaseBinder`. Does not support DCX sources.""" if isinstance(binder_source, (str, Path)): binder_path = Path(binder_source) if binder_path.is_file() and binder_path.name == "binder_manifest.json": binder_path = binder_path.parent if binder_path.is_dir(): try: with (binder_path / "binder_manifest.json").open("rb") as f: return json.load(f)["version"] == cls.__name__ # "BND3" or "BND4" except FileNotFoundError: return False elif binder_path.is_file(): reader = BinaryReader(binder_path) try: version = reader.unpack_string(length=4, encoding="ascii") except ValueError: return False if version[:3] in {"BHF", "BDF"}: version = f"BXF{version[3]}" # BXF header or data file return version == cls.__name__ return False elif isinstance(binder_source, (bytes, io.BufferedIOBase)): binder_source = BinaryReader(binder_source) if isinstance(binder_source, BinaryReader): with binder_source.temp_offset(0): try: version = binder_source.unpack_string(length=4, encoding="ascii") except ValueError: return False if version[:3] in {"BHF", "BDF"}: version = f"BXF{version[3]}" # BXF header or data file return version == cls.__name__ raise TypeError(f"Cannot detect `Binder` class from source type: {binder_source}")
def unpack(self, msb_reader: BinaryReader): header = msb_reader.unpack_struct(self.MAP_ENTITY_LIST_HEADER) entry_offsets = [ msb_reader.unpack_struct( self.MAP_ENTITY_ENTRY_OFFSET)["entry_offset"] for _ in range(header["entry_offset_count"] - 1) # 'entry_offset_count' includes tail offset ] next_entry_list_offset = msb_reader.unpack_struct( self.MAP_ENTITY_LIST_TAIL)["next_entry_list_offset"] name = msb_reader.unpack_string(offset=header["name_offset"], encoding=self.NAME_ENCODING) if name != self.INTERNAL_NAME: raise ValueError( f"MSB entry list internal name '{name}' does not match known name '{self.INTERNAL_NAME}'." ) self._entries = [] for entry_offset in entry_offsets: msb_reader.seek(entry_offset) entry = self.ENTRY_CLASS(msb_reader) self._entries.append(entry) msb_reader.seek(next_entry_list_offset)
def read(reader: BinaryReader, size: int): return reader.unpack_string(length=size // 2, encoding="utf-16-le")
def read(reader: BinaryReader, size: int): return reader.unpack_string(length=size, encoding="shift_jis_2004")
def unpack_from( cls, reader: BinaryReader, platform: TPFPlatform, tpf_flags: int, encoding: str, tpf_path: tp.Union[None, str, Path] = None, ): self = cls() self.tpf_path = tpf_path file_offset = reader.unpack_value("I") file_size = reader.unpack_value("i") self.format = reader.unpack_value("B") self.texture_type = TextureType(reader.unpack_value("B")) self.mipmaps = reader.unpack_value("B") self.texture_flags = reader.unpack_value("B") if self.texture_flags not in {0, 1, 2, 3}: raise ValueError( f"`TPFTexture.flags1` was {self.texture_flags}, but expected 0, 1, 2, or 3." ) if platform != TPFPlatform.PC: self.header = TextureHeader self.header.width = reader.unpack_value("h") self.header.height = reader.unpack_value("h") if platform == TPFPlatform.Xbox360: reader.assert_pad(4) elif platform == TPFPlatform.PS3: self.header.unk1 = reader.unpack_value("i") if tpf_flags != 0: self.header.unk2 = reader.unpack_value("i") if self.header.unk2 not in {0, 0x68E0, 0xAAE4}: raise ValueError( f"`TextureHeader.unk2` was {self.header.unk2}, but expected 0, 0x68E0, or 0xAAE4." ) elif platform in {TPFPlatform.PS4, TPFPlatform.XboxOne}: self.header.texture_count = reader.unpack_value("i") if self.header.texture_count not in {1, 6}: f"`TextureHeader.texture_count` was {self.header.texture_count}, but expected 1 or 6." self.header.unk2 = reader.unpack_value("i") if self.header.unk2 != 0xD: f"`TextureHeader.unk2` was {self.header.unk2}, but expected 0xD." name_offset = reader.unpack_value("I") has_float_struct = reader.unpack_value("i") == 1 if platform in {TPFPlatform.PS4, TPFPlatform.XboxOne}: self.header.dxgi_format = reader.unpack_value("i") if has_float_struct: self.float_struct = FloatStruct.unpack_from(reader) with reader.temp_offset(file_offset): self.data = reader.read(file_size) if self.texture_flags in {2, 3}: # Data is DCX-compressed. # TODO: should enforce DCX type as 'DCP_EDGE'? self.data = decompress(self.data) self.name = reader.unpack_string(offset=name_offset, encoding=encoding) return self
def unpack(self, reader: BinaryReader, **kwargs): self.byte_order = reader.byte_order = ">" if reader.unpack_value( "B", offset=44) == 255 else "<" version_info = reader.unpack("bbb", offset=45) self.flags1 = ParamFlags1(version_info[0]) self.flags2 = ParamFlags2(version_info[1]) self.paramdef_format_version = version_info[2] header_struct = self.GET_HEADER_STRUCT(self.flags1, self.byte_order) header = reader.unpack_struct(header_struct) try: self.param_type = header["param_type"] except KeyError: self.param_type = reader.unpack_string( offset=header["param_type_offset"], encoding="utf-8") self.paramdef_data_version = header["paramdef_data_version"] self.unknown = header["unknown"] # Row data offset in header not used. (It's an unsigned short, yet doesn't limit row count to 5461.) name_data_offset = header[ "name_data_offset"] # CANNOT BE TRUSTED IN VANILLA FILES! Off by +12 bytes. # Load row pointer data. row_struct = self.ROW_STRUCT_64 if self.flags1.LongDataOffset else self.ROW_STRUCT_32 row_pointers = reader.unpack_structs(row_struct, count=header["row_count"]) row_data_offset = reader.position # Reliable row data offset. # Row size is lazily determined. TODO: Unpack row data in sequence and associate with names separately. if len(row_pointers) == 0: return elif len(row_pointers) == 1: # NOTE: The only vanilla param in Dark Souls with one row is LEVELSYNC_PARAM_ST (Remastered only), # for which the row size is hard-coded here. Otherwise, we can trust the repacked offset from Soulstruct # (and SoulsFormats, etc.). if self.param_type == "LEVELSYNC_PARAM_ST": row_size = 220 else: row_size = name_data_offset - row_data_offset else: row_size = row_pointers[1]["data_offset"] - row_pointers[0][ "data_offset"] # Note that we no longer need to track reader offset. name_encoding = self.get_name_encoding() for row_struct in row_pointers: reader.seek(row_struct["data_offset"]) row_data = reader.read(row_size) if row_struct["name_offset"] != 0: try: name = reader.unpack_string( offset=row_struct["name_offset"], encoding=name_encoding, reset_old_offset=False, # no need to reset ) except UnicodeDecodeError as ex: if ex.object in self.undecodable_row_names: name = reader.unpack_bytes( offset=row_struct["name_offset"], reset_old_offset=False, # no need to reset ) else: raise except ValueError: reader.seek(row_struct["name_offset"]) _LOGGER.error( f"Error encountered while parsing row name string in {self.param_type}.\n" f" Header: {header}\n" f" Row Struct: {row_struct}\n" f" 30 chrs of name data: {' '.join(f'{{:02x}}'.format(x) for x in reader.read(30))}" ) raise else: name = "" self.rows[row_struct["id"]] = ParamRow(row_data, self.paramdef, name=name)