def unpack(self, msb_reader: BinaryReader):
part_offset = msb_reader.position
header = msb_reader.unpack_struct(self.PART_HEADER_STRUCT)
if header["__part_type"] != self.ENTRY_SUBTYPE:
raise ValueError(f"Unexpected part type enum {header['part_type']} for class {self.__class__.__name__}.")
self._instance_index = header["_instance_index"]
self._model_index = header["_model_index"]
self._part_type_index = header["_part_type_index"]
for transform in ("translate", "rotate", "scale"):
setattr(self, transform, Vector3(header[transform]))
self._draw_groups = int_group_to_bit_set(header["__draw_groups"], assert_size=8)
self._display_groups = int_group_to_bit_set(header["__display_groups"], assert_size=8)
self._backread_groups = int_group_to_bit_set(header["__backread_groups"], assert_size=8)
self.description = msb_reader.unpack_string(
offset=part_offset + header["__description_offset"], encoding="utf-16-le",
)
self.name = msb_reader.unpack_string(
offset=part_offset + header["__name_offset"], encoding="utf-16-le",
)
self.sib_path = msb_reader.unpack_string(
offset=part_offset + header["__sib_path_offset"], encoding="utf-16-le",
)
msb_reader.seek(part_offset + header["__base_data_offset"])
base_data = msb_reader.unpack_struct(self.PART_BASE_DATA_STRUCT)
self.set(**base_data)
msb_reader.seek(part_offset + header["__type_data_offset"])
self.unpack_type_data(msb_reader)
self._unpack_gparam_data(msb_reader, part_offset, header)
self._unpack_scene_gparam_data(msb_reader, part_offset, header)
def unpack(self, msb_reader: BinaryReader):
model_offset = msb_reader.position
model_data = msb_reader.unpack_struct(self.MODEL_STRUCT)
self.name = msb_reader.unpack_string(
offset=model_offset + model_data["__name_offset"], encoding=self.NAME_ENCODING
)
self.sib_path = msb_reader.unpack_string(
offset=model_offset + model_data["__sib_path_offset"], encoding=self.NAME_ENCODING,
)
try:
self.ENTRY_SUBTYPE = MSBModelSubtype(model_data["__model_type"])
except TypeError:
raise ValueError(f"Unrecognized MSB model type: {model_data['__model_type']}")
self.set(**model_data)
def unpack(self, msb_reader: BinaryReader):
model_offset = msb_reader.position
header = msb_reader.unpack_struct(self.MODEL_STRUCT)
self.name = msb_reader.unpack_string(offset=model_offset +
header["__name_offset"],
encoding=self.NAME_ENCODING)
self.sib_path = msb_reader.unpack_string(
offset=model_offset + header["__sib_path_offset"],
encoding=self.NAME_ENCODING,
)
if header["__model_type"] != self.ENTRY_SUBTYPE.value:
raise ValueError(
f"Unexpected MSB model type value {header['__model_type']} for {self.__class__.__name__}. "
f"Expected {self.ENTRY_SUBTYPE.value}.")
self.set(**header)
def unpack(self, reader: BinaryReader, remove_empty_entries=True):
header = reader.unpack_struct(self.HEADER_STRUCT)
# Groups of contiguous text string IDs are defined by ranges (first ID, last ID) to save space.
ranges = reader.unpack_structs(self.RANGE_STRUCT,
count=header["range_count"])
if reader.position != header["string_offsets_offset"]:
_LOGGER.warning(
"Range data did not end at string data offset given in FMG header."
)
string_offsets = reader.unpack_structs(self.STRING_OFFSET_STRUCT,
count=header["string_count"])
# Text pointer table corresponds to all the IDs (joined together) of the above ranges, in order.
for string_range in ranges:
i = string_range["first_index"]
for string_id in range(string_range["first_id"],
string_range["last_id"] + 1):
if string_id in self.entries:
raise ValueError(
f"Malformed FMG: Entry index {string_id} appeared more than once."
)
string_offset = string_offsets[i]["offset"]
if string_offset == 0:
if not remove_empty_entries:
# Empty text string. These will trigger in-game error messages, like ?PlaceName?.
# Distinct from ' ', which is intentionally blank text data (e.g. the unused area subtitles).
self.entries[string_id] = ""
else:
string = reader.unpack_string(offset=string_offset,
encoding="utf-16le")
if string or not remove_empty_entries:
self.entries[string_id] = string
i += 1
def unpack(self, reader: BinaryReader, start_offset: int, depth=0):
data = reader.unpack_struct(self.STRUCT)
name_length = data.pop("__name_length")
self.name = reader.unpack_string(length=name_length, encoding="ascii")
self.size = self.STRUCT.size + name_length
self.depth = depth
# TODO: Use `_properties` and `field` properties, which inspect the node name, etc.
self._properties = [FBXProperty.unpack(reader) for _ in range(data.pop("__property_count"))]
self.size += data.pop("__property_list_size")
self.children = []
end_offset = data.pop("__end_offset")
while start_offset + self.size < end_offset:
child = self.__class__(reader, start_offset=start_offset + self.size, depth=self.depth + 1)
self.size += child.size
if start_offset + self.size == end_offset:
break # empty node is not kept
self.children.append(child)
if self.name == "P":
if self.children:
raise ValueError("`FBXNode` named 'P' should not have any children.")
name, *args = [p.value for p in self._properties]
self._field = FBXPropertyField(name, *args)
else:
self._field = None
def unpack(self, esd_reader: BinaryReader, **kwargs):
header = esd_reader.unpack_struct(self.EXTERNAL_HEADER_STRUCT)
# Internal offsets start here, so we reset the buffer.
esd_reader = BinaryReader(esd_reader.read())
internal_header = esd_reader.unpack_struct(self.INTERNAL_HEADER_STRUCT)
self.magic = internal_header["magic"]
state_machine_headers = esd_reader.unpack_structs(
self.STATE_MACHINE_HEADER_STRUCT,
count=header["state_machine_count"])
for state_machine_header in state_machine_headers:
states = self.State.unpack(
esd_reader,
state_machine_header["state_machine_offset"],
count=state_machine_header["state_count"],
)
self.state_machines[
state_machine_header["state_machine_index"]] = states
if internal_header["esd_name_length"] > 0:
esd_name_offset = internal_header["esd_name_offset"]
esd_name_length = internal_header["esd_name_length"]
# Note the given length is the length of the final string. The actual UTF-16 encoded bytes are twice that.
self.esd_name = esd_reader.unpack_string(offset=esd_name_offset,
length=2 *
esd_name_length,
encoding="utf-16le")
esd_reader.seek(esd_name_offset + 2 * esd_name_length)
self.file_tail = esd_reader.read()
else:
self.esd_name = ""
esd_reader.seek(header["unk_offset_1"]) # after packed EZL
self.file_tail = esd_reader.read()
def unpack_strings(self) -> list[tuple[str, str]]:
strings = []
string_reader = BinaryReader(self.packed_strings)
while string_reader.position != len(self.packed_strings):
offset = string_reader.position
string = string_reader.unpack_string(encoding=self.STRING_ENCODING)
strings.append((str(offset), string))
return strings
def unpack_goal(self, reader: BinaryReader,
goal_struct: BinaryStruct) -> LuaGoal:
goal = reader.unpack_struct(goal_struct,
byte_order=">" if self.big_endian else "<")
name = reader.unpack_string(offset=goal["name_offset"],
encoding=self.encoding)
if goal["logic_interrupt_name_offset"] > 0:
logic_interrupt_name = reader.unpack_string(
offset=goal["logic_interrupt_name_offset"],
encoding=self.encoding)
else:
logic_interrupt_name = ""
return LuaGoal(
goal_id=goal["goal_id"],
goal_name=name,
has_battle_interrupt=goal["has_battle_interrupt"],
has_logic_interrupt=goal["has_logic_interrupt"],
logic_interrupt_name=logic_interrupt_name,
)
def unpack(
self,
reader: BinaryReader,
encoding: str,
version: Version,
gx_lists: tp.List[GXList],
gx_list_indices: tp.Dict[int, int],
):
material = reader.unpack_struct(self.STRUCT)
self.name = reader.unpack_string(offset=material.pop("__name__z"), encoding=encoding)
self.mtd_path = reader.unpack_string(offset=material.pop("__mtd_path__z"), encoding=encoding)
gx_offset = material.pop("__gx_offset")
if gx_offset == 0:
self.gx_index = -1
elif gx_offset in gx_list_indices:
self.gx_index = gx_list_indices[gx_offset]
else:
self.gx_index = gx_list_indices[gx_offset] = len(gx_lists)
with reader.temp_offset(gx_offset):
gx_lists.append(GXList(reader, version))
self.set(**material)
def unpack(self, reader: BinaryReader, **kwargs):
self.big_endian, self.use_struct_64 = self._check_big_endian_and_struct_64(
reader)
fmt = f"{'>' if self.big_endian else '<'}{'q' if self.use_struct_64 else 'i'}"
read_size = struct.calcsize(fmt)
self.names = []
offset = None
while offset != 0:
(offset, ) = struct.unpack(fmt, reader.read(read_size))
if offset != 0:
self.names.append(
reader.unpack_string(offset=offset,
encoding=self.encoding))
def unpack(self, msb_reader: BinaryReader):
event_offset = msb_reader.position
header = msb_reader.unpack_struct(self.EVENT_HEADER_STRUCT)
if header["__event_type"] != self.ENTRY_SUBTYPE:
raise ValueError(f"Unexpected MSB event type value {header['__event_type']} for {self.__class__.__name__}.")
msb_reader.seek(event_offset + header["__base_data_offset"])
base_data = msb_reader.unpack_struct(self.EVENT_BASE_DATA_STRUCT)
name_offset = event_offset + header["__name_offset"]
self.name = msb_reader.unpack_string(offset=name_offset, encoding=self.NAME_ENCODING)
self.set(**header)
self.set(**base_data)
msb_reader.seek(event_offset + header["__type_data_offset"])
self.unpack_type_data(msb_reader)
def unpack_fields(
cls,
param_name: str,
paramdef_reader: BinaryReader,
field_count: int,
format_version: int,
unicode: bool,
byte_order: str,
) -> dict[str, ParamDefField]:
"""Buffer should be at the start of the packed fields (which are followed by the packed descriptions)."""
field_structs = paramdef_reader.unpack_structs(cls.GET_FIELD_STRUCT(
format_version, unicode, byte_order),
count=field_count)
fields = {}
for field_index, field_struct in enumerate(field_structs):
if field_struct["description_offset"] != 0:
field_description = paramdef_reader.unpack_string(
offset=field_struct["description_offset"],
encoding="utf-16-le" if unicode else "shift_jis_2004",
)
else:
field_description = ""
if "display_name_offset" in field_struct:
display_name = paramdef_reader.unpack_string(
offset=field_struct["display_name_offset"],
encoding="utf-16-le",
)
else:
display_name = field_struct["display_name"]
field = cls(field_struct,
field_index,
field_description,
param_name,
display_name=display_name)
fields[field.name] = field
return fields
def detect(cls, binder_source: tp.Union[GameFile.Typing, dict]) -> bool:
"""Returns True if `binder_source` appears to be this subclass of `BaseBinder`. Does not support DCX sources."""
if isinstance(binder_source, dict):
# Manifest dictionary. Simply check version.
return binder_source.get(
"version") == cls.__name__ # "BND3", "BND4", etc.
if isinstance(binder_source, (str, Path)):
binder_path = Path(binder_source)
if binder_path.is_file(
) and binder_path.name == "binder_manifest.json":
binder_path = binder_path.parent
if binder_path.is_dir():
try:
manifest = read_json(binder_path / "binder_manifest.json",
encoding="shift_jis")
return manifest.get(
"version") == cls.__name__ # "BND3", "BND4", etc.
except FileNotFoundError:
return False
elif binder_path.is_file():
reader = BinaryReader(binder_path)
try:
version = reader.unpack_string(length=4, encoding="ascii")
except ValueError:
return False
if version[:3] in {"BHF", "BDF"}:
version = f"BXF{version[3]}" # BXF header or data file
return version == cls.__name__
return False
elif isinstance(binder_source, (bytes, io.BufferedIOBase)):
binder_source = BinaryReader(binder_source)
if isinstance(binder_source, BinaryReader):
with binder_source.temp_offset(0):
try:
version = binder_source.unpack_string(length=4,
encoding="ascii")
except ValueError:
return False
if version[:3] in {"BHF", "BDF"}:
version = f"BXF{version[3]}" # BXF header or data file
return version == cls.__name__
raise TypeError(
f"Cannot detect `Binder` class from source type: {binder_source}")
def unpack(self, paramdef_reader: BinaryReader, **kwargs):
header = paramdef_reader.unpack_struct(self.HEADER_STRUCT)
if "param_name" in header:
self.param_type = header["param_name"]
else:
self.param_type = paramdef_reader.unpack_string(
offset=header["param_name_offset"],
encoding="shift_jis_2004", # never unicode
)
self.data_version = header["data_version"]
self.format_version = header["format_version"]
self.unicode = header["unicode"]
self.fields = self.FIELD_CLASS.unpack_fields(
self.param_type,
paramdef_reader,
header["field_count"],
self.format_version,
self.unicode,
self.BYTE_ORDER,
)
def from_bnd3_reader(cls, reader: BinaryReader, binder_flags: BinderFlags, bit_big_endian: bool):
flags = BinderEntryFlags.read(reader, bit_big_endian)
reader.assert_pad(3)
compressed_size = reader.unpack_value("i")
data_offset = reader.unpack_value("q" if binder_flags.has_long_offsets else "I")
entry_id = reader.unpack_value("i") if binder_flags.has_ids else None
if binder_flags.has_names:
path_offset = reader.unpack_value("i")
path = reader.unpack_string(path_offset, encoding="shift-jis") # NOT `shift_jis_2004`
else:
path = None
uncompressed_size = reader.unpack_value("i") if binder_flags.has_compression else None
return cls(
flags=flags,
compressed_size=compressed_size,
entry_id=entry_id,
path=path,
uncompressed_size=uncompressed_size,
data_offset=data_offset,
)
def from_bnd4_reader(cls, reader: BinaryReader, binder_flags: BinderFlags, bit_big_endian: bool, unicode: bool):
flags = BinderEntryFlags.read(reader, bit_big_endian)
reader.assert_pad(3)
assert reader.unpack_value("i") == -1
compressed_size = reader.unpack_value("q")
uncompressed_size = reader.unpack_value("q") if binder_flags.has_compression else None
data_offset = reader.unpack_value("q" if binder_flags.has_long_offsets else "I")
entry_id = reader.unpack_value("i") if binder_flags.has_ids else None
if binder_flags.has_names:
path_offset = reader.unpack_value("I")
path = reader.unpack_string(path_offset, encoding="utf-16-le" if unicode else "shift-jis")
else:
path = None
return cls(
flags=flags,
compressed_size=compressed_size,
entry_id=entry_id,
path=path,
uncompressed_size=uncompressed_size,
data_offset=data_offset,
)
def unpack(self, msb_reader: BinaryReader):
header = msb_reader.unpack_struct(self.MAP_ENTITY_LIST_HEADER)
entry_offsets = [
msb_reader.unpack_struct(
self.MAP_ENTITY_ENTRY_OFFSET)["entry_offset"]
for _ in range(header["entry_offset_count"] -
1) # 'entry_offset_count' includes tail offset
]
next_entry_list_offset = msb_reader.unpack_struct(
self.MAP_ENTITY_LIST_TAIL)["next_entry_list_offset"]
self.name = msb_reader.unpack_string(offset=header["name_offset"],
encoding=self.NAME_ENCODING)
self._entries = []
for entry_offset in entry_offsets:
msb_reader.seek(entry_offset)
entry = self.ENTRY_CLASS(msb_reader)
self._entries.append(entry)
msb_reader.seek(next_entry_list_offset)
def unpack(self, msb_reader: BinaryReader):
region_offset = msb_reader.position
base_data = msb_reader.unpack_struct(self.REGION_STRUCT)
self.name = msb_reader.unpack_string(
offset=region_offset + base_data["name_offset"],
encoding=self.NAME_ENCODING,
)
self._region_index = base_data["__region_index"]
self.translate = Vector3(base_data["translate"])
self.rotate = Vector3(base_data["rotate"])
self.check_null_field(msb_reader,
region_offset + base_data["unknown_offset_1"])
self.check_null_field(msb_reader,
region_offset + base_data["unknown_offset_2"])
if base_data["type_data_offset"] != 0:
msb_reader.seek(region_offset + base_data["type_data_offset"])
self.unpack_type_data(msb_reader)
msb_reader.seek(region_offset + base_data["entity_id_offset"])
self.entity_id = msb_reader.unpack_value("i")
return region_offset + base_data["entity_id_offset"]
def detect(cls, binder_source: GameFile.Typing) -> bool:
"""Returns True if `binder_source` appears to be this subclass of `BaseBinder`. Does not support DCX sources."""
if isinstance(binder_source, (str, Path)):
binder_path = Path(binder_source)
if binder_path.is_file() and binder_path.name == "binder_manifest.json":
binder_path = binder_path.parent
if binder_path.is_dir():
try:
with (binder_path / "binder_manifest.json").open("rb") as f:
return json.load(f)["version"] == cls.__name__ # "BND3" or "BND4"
except FileNotFoundError:
return False
elif binder_path.is_file():
reader = BinaryReader(binder_path)
try:
version = reader.unpack_string(length=4, encoding="ascii")
except ValueError:
return False
if version[:3] in {"BHF", "BDF"}:
version = f"BXF{version[3]}" # BXF header or data file
return version == cls.__name__
return False
elif isinstance(binder_source, (bytes, io.BufferedIOBase)):
binder_source = BinaryReader(binder_source)
if isinstance(binder_source, BinaryReader):
with binder_source.temp_offset(0):
try:
version = binder_source.unpack_string(length=4, encoding="ascii")
except ValueError:
return False
if version[:3] in {"BHF", "BDF"}:
version = f"BXF{version[3]}" # BXF header or data file
return version == cls.__name__
raise TypeError(f"Cannot detect `Binder` class from source type: {binder_source}")
def unpack(self, msb_reader: BinaryReader):
header = msb_reader.unpack_struct(self.MAP_ENTITY_LIST_HEADER)
entry_offsets = [
msb_reader.unpack_struct(
self.MAP_ENTITY_ENTRY_OFFSET)["entry_offset"]
for _ in range(header["entry_offset_count"] -
1) # 'entry_offset_count' includes tail offset
]
next_entry_list_offset = msb_reader.unpack_struct(
self.MAP_ENTITY_LIST_TAIL)["next_entry_list_offset"]
name = msb_reader.unpack_string(offset=header["name_offset"],
encoding=self.NAME_ENCODING)
if name != self.INTERNAL_NAME:
raise ValueError(
f"MSB entry list internal name '{name}' does not match known name '{self.INTERNAL_NAME}'."
)
self._entries = []
for entry_offset in entry_offsets:
msb_reader.seek(entry_offset)
entry = self.ENTRY_CLASS(msb_reader)
self._entries.append(entry)
msb_reader.seek(next_entry_list_offset)
def read(reader: BinaryReader, size: int):
return reader.unpack_string(length=size // 2, encoding="utf-16-le")
def read(reader: BinaryReader, size: int):
return reader.unpack_string(length=size, encoding="shift_jis_2004")
def unpack_from(
cls,
reader: BinaryReader,
platform: TPFPlatform,
tpf_flags: int,
encoding: str,
tpf_path: tp.Union[None, str, Path] = None,
):
self = cls()
self.tpf_path = tpf_path
file_offset = reader.unpack_value("I")
file_size = reader.unpack_value("i")
self.format = reader.unpack_value("B")
self.texture_type = TextureType(reader.unpack_value("B"))
self.mipmaps = reader.unpack_value("B")
self.texture_flags = reader.unpack_value("B")
if self.texture_flags not in {0, 1, 2, 3}:
raise ValueError(
f"`TPFTexture.flags1` was {self.texture_flags}, but expected 0, 1, 2, or 3."
)
if platform != TPFPlatform.PC:
self.header = TextureHeader
self.header.width = reader.unpack_value("h")
self.header.height = reader.unpack_value("h")
if platform == TPFPlatform.Xbox360:
reader.assert_pad(4)
elif platform == TPFPlatform.PS3:
self.header.unk1 = reader.unpack_value("i")
if tpf_flags != 0:
self.header.unk2 = reader.unpack_value("i")
if self.header.unk2 not in {0, 0x68E0, 0xAAE4}:
raise ValueError(
f"`TextureHeader.unk2` was {self.header.unk2}, but expected 0, 0x68E0, or 0xAAE4."
)
elif platform in {TPFPlatform.PS4, TPFPlatform.XboxOne}:
self.header.texture_count = reader.unpack_value("i")
if self.header.texture_count not in {1, 6}:
f"`TextureHeader.texture_count` was {self.header.texture_count}, but expected 1 or 6."
self.header.unk2 = reader.unpack_value("i")
if self.header.unk2 != 0xD:
f"`TextureHeader.unk2` was {self.header.unk2}, but expected 0xD."
name_offset = reader.unpack_value("I")
has_float_struct = reader.unpack_value("i") == 1
if platform in {TPFPlatform.PS4, TPFPlatform.XboxOne}:
self.header.dxgi_format = reader.unpack_value("i")
if has_float_struct:
self.float_struct = FloatStruct.unpack_from(reader)
with reader.temp_offset(file_offset):
self.data = reader.read(file_size)
if self.texture_flags in {2, 3}:
# Data is DCX-compressed.
# TODO: should enforce DCX type as 'DCP_EDGE'?
self.data = decompress(self.data)
self.name = reader.unpack_string(offset=name_offset, encoding=encoding)
return self
def unpack(self, reader: BinaryReader, **kwargs):
self.byte_order = reader.byte_order = ">" if reader.unpack_value(
"B", offset=44) == 255 else "<"
version_info = reader.unpack("bbb", offset=45)
self.flags1 = ParamFlags1(version_info[0])
self.flags2 = ParamFlags2(version_info[1])
self.paramdef_format_version = version_info[2]
header_struct = self.GET_HEADER_STRUCT(self.flags1, self.byte_order)
header = reader.unpack_struct(header_struct)
try:
self.param_type = header["param_type"]
except KeyError:
self.param_type = reader.unpack_string(
offset=header["param_type_offset"], encoding="utf-8")
self.paramdef_data_version = header["paramdef_data_version"]
self.unknown = header["unknown"]
# Row data offset in header not used. (It's an unsigned short, yet doesn't limit row count to 5461.)
name_data_offset = header[
"name_data_offset"] # CANNOT BE TRUSTED IN VANILLA FILES! Off by +12 bytes.
# Load row pointer data.
row_struct = self.ROW_STRUCT_64 if self.flags1.LongDataOffset else self.ROW_STRUCT_32
row_pointers = reader.unpack_structs(row_struct,
count=header["row_count"])
row_data_offset = reader.position # Reliable row data offset.
# Row size is lazily determined. TODO: Unpack row data in sequence and associate with names separately.
if len(row_pointers) == 0:
return
elif len(row_pointers) == 1:
# NOTE: The only vanilla param in Dark Souls with one row is LEVELSYNC_PARAM_ST (Remastered only),
# for which the row size is hard-coded here. Otherwise, we can trust the repacked offset from Soulstruct
# (and SoulsFormats, etc.).
if self.param_type == "LEVELSYNC_PARAM_ST":
row_size = 220
else:
row_size = name_data_offset - row_data_offset
else:
row_size = row_pointers[1]["data_offset"] - row_pointers[0][
"data_offset"]
# Note that we no longer need to track reader offset.
name_encoding = self.get_name_encoding()
for row_struct in row_pointers:
reader.seek(row_struct["data_offset"])
row_data = reader.read(row_size)
if row_struct["name_offset"] != 0:
try:
name = reader.unpack_string(
offset=row_struct["name_offset"],
encoding=name_encoding,
reset_old_offset=False, # no need to reset
)
except UnicodeDecodeError as ex:
if ex.object in self.undecodable_row_names:
name = reader.unpack_bytes(
offset=row_struct["name_offset"],
reset_old_offset=False, # no need to reset
)
else:
raise
except ValueError:
reader.seek(row_struct["name_offset"])
_LOGGER.error(
f"Error encountered while parsing row name string in {self.param_type}.\n"
f" Header: {header}\n"
f" Row Struct: {row_struct}\n"
f" 30 chrs of name data: {' '.join(f'{{:02x}}'.format(x) for x in reader.read(30))}"
)
raise
else:
name = ""
self.rows[row_struct["id"]] = ParamRow(row_data,
self.paramdef,
name=name)
