def unpack_value(self, reader: BinaryReader) -> PropertyValueTyping:
if self in {self.Bytes, self.String}:
raw_data_size = reader.unpack_value("i")
value = reader.read(raw_data_size)
if self == PropertyType.String:
# Assumed encoding (could also be "ascii").
return value.decode("utf-8") # str
return value # bytes
if "Array" not in self.name:
return reader.unpack_value(self.get_fmt())
array_length, is_compressed, compressed_size = reader.unpack("III")
if is_compressed:
decompressed_size = self.get_size(array=True) * array_length
decompressed = zlib.decompressobj().decompress(reader.read(compressed_size))
if len(decompressed) != decompressed_size:
raise ValueError(
f"FBX property decompressed data size ({len(decompressed)}) does not match expected size "
f"({decompressed_size})"
)
array_reader = BinaryReader(decompressed)
else:
array_reader = reader
fmt = self.get_fmt(array_length)
return list(array_reader.unpack(fmt))
def unpack(self,
reader: BinaryReader,
bounding_box_has_unknown: bool = None):
mesh = reader.unpack_struct(self.STRUCT)
bounding_box_offset = mesh.pop("__bounding_box_offset")
if bounding_box_offset == 0:
self.bounding_box = None
else:
with reader.temp_offset(bounding_box_offset):
self.bounding_box = BoundingBoxWithUnknown(
reader) if bounding_box_has_unknown else BoundingBox(
reader)
bone_count = mesh.pop("__bone_count")
with reader.temp_offset(mesh.pop("__bone_offset")):
self.bone_indices = list(reader.unpack(f"<{bone_count}i"))
face_set_count = mesh.pop("__face_set_count")
with reader.temp_offset(mesh.pop("__face_set_offset")):
self._face_set_indices = list(reader.unpack(f"<{face_set_count}i"))
vertex_count = mesh.pop("__vertex_buffer_count")
with reader.temp_offset(mesh.pop("__vertex_buffer_offset")):
self._vertex_buffer_indices = list(
reader.unpack(f"<{vertex_count}i"))
self.set(**mesh)
def get_instruction_args(reader: BinaryReader, category, index,
first_arg_offset, event_args_size, emedf: dict):
"""Process instruction arguments (required and optional) from EMEVD binary."""
try:
emedf_args_info = emedf[category, index]["args"]
except KeyError:
raise KeyError(
f"Could not find instruction ({category}, {index}) in `Instruction.EMEDF`."
)
previous_offset = reader.position
if event_args_size == 0:
return "", []
try:
args_format = "@" + "".join(arg["internal_type"].get_fmt()
for arg in emedf_args_info.values())
except KeyError:
raise KeyError(
f"Cannot find argument types for instruction {category}[{index:02d}] ({event_args_size} bytes)"
)
# 's' arguments are actually four-byte offsets into the packed string data, though we will keep the 's' symbol.
struct_args_format = args_format.replace("s", "I")
required_args_size = struct.calcsize(struct_args_format)
if required_args_size > event_args_size:
raise ValueError(
f"Documented size of minimum required args for instruction {category}"
f"[{index}] is {required_args_size}, but size of args specified in EMEVD file is "
f"only {event_args_size}.")
reader.seek(first_arg_offset)
args = reader.unpack(struct_args_format)
# Additional arguments may appear for the instruction 2000[00], 'RunEvent'. These instructions are tightly packed
# and are always aligned to 4. We read them here as unsigned integers and must actually parse the called event ID to
# interpret them properly (done at `EMEVD` class level).
extra_size = event_args_size - required_args_size
opt_arg_count = extra_size // 4
if opt_arg_count == 0:
reader.seek(previous_offset)
return args_format[1:], list(args)
elif (category, index) not in _OPTIONAL_ARGS_ALLOWED:
raise ValueError(
f"Extra arguments found for instruction {category}[{index}], which is not permitted. Arg types may be "
f"wrong (too short) for this instruction.\n"
f" required size = {required_args_size}\n"
f" actual size = {event_args_size}")
elif extra_size % 4 != 0:
raise ValueError(
f"Error interpreting instruction {category}[{index}]: optional argument "
f"size is not a multiple of four bytes ({extra_size}).")
opt_args = [reader.unpack_value("<I") for _ in range(opt_arg_count)]
reader.seek(previous_offset)
return args_format[1:] + "|" + "I" * (extra_size //
4), list(args) + opt_args
def unpack_from(cls, reader: BinaryReader):
float_struct = cls()
float_struct.unk0 = reader.unpack_value("i")
length = reader.unpack_value("i")
if length < 0 or length % 4:
raise ValueError(
f"Unexpected `FloatStruct` length: {length}. Expected a multiple of 4 (or 0)."
)
float_struct.values = list(reader.unpack(f"{length // 4}f"))
def ENTRY_CLASS(cls, msb_reader: BinaryReader):
"""Detects the appropriate subclass of `MSBPart` to instantiate, and does so."""
entry_subtype_int = msb_reader.unpack("i",
offset=cls.SUBTYPE_OFFSET,
relative_offset=True)[0]
try:
entry_subtype = cls.ENTRY_SUBTYPE_ENUM(entry_subtype_int)
except ValueError:
raise MapError(
f"Entry of type {cls.ENTRY_SUBTYPE_ENUM} has invalid subtype enum: {entry_subtype_int}"
)
return cls.SUBTYPE_CLASSES[entry_subtype](msb_reader)
def __init__(self, data: bytes):
reader = BinaryReader(data)
reader.unpack_value("4s", asserted=b"DDS ")
reader.unpack_value("i", asserted=0x7C)
self.flags = reader.unpack_value("I")
self.height = reader.unpack_value("i")
self.width = reader.unpack_value("i")
self.pitch_or_linear_size = reader.unpack_value("i")
self.depth = reader.unpack_value("i")
self.mipmap_count = reader.unpack_value("i")
self.reserved_1 = reader.unpack("11i")
# TODO: More here (see SoulsFormats excerpt below), but I care mainly about width/height right now.
"""
def get_instruction_args(reader: BinaryReader, category, index,
first_arg_offset, event_args_size, format_dict):
"""Process instruction arguments (required and optional) from EMEVD binary."""
previous_offset = reader.position
if event_args_size == 0:
return "", []
try:
args_format = "@" + format_dict[category][index]
except KeyError:
raise KeyError(
f"Cannot find argument types for instruction {category}[{index:02d}]."
)
# 's' arguments are actually four-byte offsets into the packed string data, though we will keep the 's' symbol.
struct_args_format = args_format.replace("s", "I")
required_args_size = struct.calcsize(struct_args_format)
if required_args_size > event_args_size:
raise ValueError(
f"Documented size of minimum required args for instruction {category}"
f"[{index}] is {required_args_size}, but size of args specified in EMEVD file is "
f"only {event_args_size}.")
reader.seek(first_arg_offset)
args = reader.unpack(struct_args_format)
# Additional arguments may appear for the instruction 2000[00], 'RunEvent'. These instructions are tightly packed
# and are always aligned to 4. We read them here as unsigned integers and must actually parse the called event ID to
# interpret them properly (done at `EMEVD` class level).
extra_size = event_args_size - required_args_size
opt_arg_count = extra_size // 4
if opt_arg_count == 0:
reader.seek(previous_offset)
return args_format[1:], list(args)
elif extra_size % 4 != 0:
raise ValueError(
f"Error interpreting instruction {category}[{index}]: optional argument "
f"size is not a multiple of four bytes ({extra_size}).")
opt_args = [reader.unpack_value("<I") for _ in range(opt_arg_count)]
reader.seek(previous_offset)
return args_format[1:] + "|" + "I" * (extra_size //
4), list(args) + opt_args
def unpack(self, reader: BinaryReader, header_vertex_index_size: int,
vertex_data_offset: int):
face_set = reader.unpack_struct(self.STRUCT)
vertex_index_size = face_set.pop("__vertex_index_size")
if vertex_index_size == 0:
vertex_index_size = header_vertex_index_size
if vertex_index_size == 8:
raise NotImplementedError(
"Soulstruct cannot support edge-compressed FLVER face sets.")
elif vertex_index_size in {16, 32}:
vertex_indices_count = face_set.pop("__vertex_indices_count")
vertex_indices_offset = face_set.pop("__vertex_indices_offset")
with reader.temp_offset(vertex_data_offset +
vertex_indices_offset):
fmt = f"<{vertex_indices_count}{'H' if vertex_index_size == 16 else 'I'}"
self.vertex_indices = list(reader.unpack(fmt))
else:
raise ValueError(
f"Unsupported face set index size: {vertex_index_size}")
self.set(**face_set)
def unpack(self, reader: BinaryReader, **kwargs):
self.byte_order = reader.byte_order = ">" if reader.unpack_value(
"B", offset=44) == 255 else "<"
version_info = reader.unpack("bbb", offset=45)
self.flags1 = ParamFlags1(version_info[0])
self.flags2 = ParamFlags2(version_info[1])
self.paramdef_format_version = version_info[2]
header_struct = self.GET_HEADER_STRUCT(self.flags1, self.byte_order)
header = reader.unpack_struct(header_struct)
try:
self.param_type = header["param_type"]
except KeyError:
self.param_type = reader.unpack_string(
offset=header["param_type_offset"], encoding="utf-8")
self.paramdef_data_version = header["paramdef_data_version"]
self.unknown = header["unknown"]
# Row data offset in header not used. (It's an unsigned short, yet doesn't limit row count to 5461.)
name_data_offset = header[
"name_data_offset"] # CANNOT BE TRUSTED IN VANILLA FILES! Off by +12 bytes.
# Load row pointer data.
row_struct = self.ROW_STRUCT_64 if self.flags1.LongDataOffset else self.ROW_STRUCT_32
row_pointers = reader.unpack_structs(row_struct,
count=header["row_count"])
row_data_offset = reader.position # Reliable row data offset.
# Row size is lazily determined. TODO: Unpack row data in sequence and associate with names separately.
if len(row_pointers) == 0:
return
elif len(row_pointers) == 1:
# NOTE: The only vanilla param in Dark Souls with one row is LEVELSYNC_PARAM_ST (Remastered only),
# for which the row size is hard-coded here. Otherwise, we can trust the repacked offset from Soulstruct
# (and SoulsFormats, etc.).
if self.param_type == "LEVELSYNC_PARAM_ST":
row_size = 220
else:
row_size = name_data_offset - row_data_offset
else:
row_size = row_pointers[1]["data_offset"] - row_pointers[0][
"data_offset"]
# Note that we no longer need to track reader offset.
name_encoding = self.get_name_encoding()
for row_struct in row_pointers:
reader.seek(row_struct["data_offset"])
row_data = reader.read(row_size)
if row_struct["name_offset"] != 0:
try:
name = reader.unpack_string(
offset=row_struct["name_offset"],
encoding=name_encoding,
reset_old_offset=False, # no need to reset
)
except UnicodeDecodeError as ex:
if ex.object in self.undecodable_row_names:
name = reader.unpack_bytes(
offset=row_struct["name_offset"],
reset_old_offset=False, # no need to reset
)
else:
raise
except ValueError:
reader.seek(row_struct["name_offset"])
_LOGGER.error(
f"Error encountered while parsing row name string in {self.param_type}.\n"
f" Header: {header}\n"
f" Row Struct: {row_struct}\n"
f" 30 chrs of name data: {' '.join(f'{{:02x}}'.format(x) for x in reader.read(30))}"
)
raise
else:
name = ""
self.rows[row_struct["id"]] = ParamRow(row_data,
self.paramdef,
name=name)
def read(self, reader: BinaryReader, layout: BufferLayout,
uv_factor: float):
self.uvs = []
self.tangents = []
self.colors = []
with reader.temp_offset(reader.position):
self.raw = reader.read(layout.get_total_size())
for member in layout:
not_implemented = False
if member.semantic == LayoutSemantic.Position:
if member.layout_type == LayoutType.Float3:
self.position = Vector3(reader.unpack("<3f"))
elif member.layout_type == LayoutType.Float4:
self.position = Vector3(reader.unpack("<3f"))[:3]
elif member.layout_type == LayoutType.EdgeCompressed:
raise NotImplementedError(
"Soulstruct cannot load FLVERs with edge-compressed vertex positions."
)
else:
not_implemented = True
elif member.semantic == LayoutSemantic.BoneWeights:
if member.layout_type == LayoutType.Byte4A:
self.bone_weights = VertexBoneWeights(
*[w / 127.0 for w in reader.unpack("<4b")])
elif member.layout_type == LayoutType.Byte4C:
self.bone_weights = VertexBoneWeights(
*[w / 255.0 for w in reader.unpack("<4B")])
elif member.layout_type in {
LayoutType.UVPair, LayoutType.Short4ToFloat4A
}:
self.bone_weights = VertexBoneWeights(
*[w / 32767.0 for w in reader.unpack("<4h")])
else:
not_implemented = True
elif member.semantic == LayoutSemantic.BoneIndices:
if member.layout_type in {
LayoutType.Byte4B, LayoutType.Byte4E
}:
self.bone_indices = VertexBoneIndices(
*reader.unpack("<4B"))
elif member.layout_type == LayoutType.ShortBoneIndices:
self.bone_indices = VertexBoneIndices(
*reader.unpack("<4h"))
else:
not_implemented = True
elif member.semantic == LayoutSemantic.Normal:
if member.layout_type == LayoutType.Float3:
self.normal = Vector3(reader.unpack("<3f"))
elif member.layout_type == LayoutType.Float4:
self.normal = Vector3(reader.unpack("<3f"))
float_normal_w = reader.unpack_value("<f")
self.normal_w = int(float_normal_w)
if self.normal_w != float_normal_w:
raise ValueError(
f"`normal_w` float was not a whole number.")
elif member.layout_type in {
LayoutType.Byte4A, LayoutType.Byte4B,
LayoutType.Byte4C, LayoutType.Byte4E
}:
self.normal = Vector3([(x - 127) / 127.0
for x in reader.unpack("<3B")])
self.normal_w = reader.unpack_value("<B")
elif member.layout_type == LayoutType.Short2toFloat2:
self.normal_w = reader.unpack_value("<B")
self.normal = Vector3(
[x / 127.0 for x in reader.unpack("<3b")])
elif member.layout_type == LayoutType.Short4ToFloat4A:
self.normal = Vector3(
[x / 32767.0 for x in reader.unpack("<3h")])
self.normal_w = reader.unpack_value("<h")
elif member.layout_type == LayoutType.Short4ToFloat4B:
self.normal = Vector3([(x - 32767) / 32767.0
for x in reader.unpack("<3H")])
self.normal_w = reader.unpack_value("<h")
else:
not_implemented = True
elif member.semantic == LayoutSemantic.UV:
if member.layout_type == LayoutType.Float2:
self.uvs.append(Vector3(*reader.unpack("<2f"), 0.0))
elif member.layout_type == LayoutType.Float3:
self.uvs.append(Vector3(*reader.unpack("<3f")))
elif member.layout_type == LayoutType.Float4:
self.uvs.append(Vector3(*reader.unpack("<2f"), 0.0))
self.uvs.append(Vector3(*reader.unpack("<2f"), 0.0))
elif member.layout_type in {
LayoutType.Byte4A, LayoutType.Byte4B,
LayoutType.Short2toFloat2, LayoutType.Byte4C,
LayoutType.UV
}:
self.uvs.append(
Vector3(*reader.unpack("<2h"), 0) / uv_factor)
elif member.layout_type == LayoutType.UVPair:
self.uvs.append(
Vector3(*reader.unpack("<2h"), 0) / uv_factor)
self.uvs.append(
Vector3(*reader.unpack("<2h"), 0) / uv_factor)
elif member.layout_type == LayoutType.Short4ToFloat4B:
self.uvs.append(Vector3(*reader.unpack("<3h")) / uv_factor)
if reader.unpack_value("<h") != 0:
raise ValueError(
"Expected zero short after reading UV | Short4ToFloat4B vertex member."
)
else:
not_implemented = True
elif member.semantic == LayoutSemantic.Tangent:
if member.layout_type == LayoutType.Float4:
self.tangents.append(Vector4(*reader.unpack("<4f")))
elif member.layout_type in {
LayoutType.Byte4A,
LayoutType.Byte4B,
LayoutType.Byte4C,
LayoutType.Short4ToFloat4A,
LayoutType.Byte4E,
}:
tangent = Vector4([(x - 127) / 127.0
for x in reader.unpack("<4B")])
self.tangents.append(tangent)
else:
not_implemented = True
elif member.semantic == LayoutSemantic.Bitangent:
if member.layout_type in {
LayoutType.Byte4A, LayoutType.Byte4B,
LayoutType.Byte4C, LayoutType.Byte4E
}:
self.bitangent = Vector4([(x - 127) / 127.0
for x in reader.unpack("<4B")])
else:
not_implemented = True
elif member.semantic == LayoutSemantic.VertexColor:
if member.layout_type == LayoutType.Float4:
self.colors.append(ColorRGBA(*reader.unpack("<4f")))
elif member.layout_type in {
LayoutType.Byte4A, LayoutType.Byte4C
}:
# Convert byte channnels [0-255] to float channels [0-1].
self.colors.append(
ColorRGBA(*[b / 255.0 for b in reader.unpack("<4B")]))
else:
not_implemented = True
else:
not_implemented = True
if not_implemented:
raise NotImplementedError(
f"Unsupported vertex member semantic/type combination: "
f"{member.semantic.name} | {member.layout_type.name}")
