def unpack_value(self, reader: BinaryReader) -> PropertyValueTyping: if self in {self.Bytes, self.String}: raw_data_size = reader.unpack_value("i") value = reader.read(raw_data_size) if self == PropertyType.String: # Assumed encoding (could also be "ascii"). return value.decode("utf-8") # str return value # bytes if "Array" not in self.name: return reader.unpack_value(self.get_fmt()) array_length, is_compressed, compressed_size = reader.unpack("III") if is_compressed: decompressed_size = self.get_size(array=True) * array_length decompressed = zlib.decompressobj().decompress(reader.read(compressed_size)) if len(decompressed) != decompressed_size: raise ValueError( f"FBX property decompressed data size ({len(decompressed)}) does not match expected size " f"({decompressed_size})" ) array_reader = BinaryReader(decompressed) else: array_reader = reader fmt = self.get_fmt(array_length) return list(array_reader.unpack(fmt))
def unpack(self, reader: BinaryReader, bounding_box_has_unknown: bool = None): mesh = reader.unpack_struct(self.STRUCT) bounding_box_offset = mesh.pop("__bounding_box_offset") if bounding_box_offset == 0: self.bounding_box = None else: with reader.temp_offset(bounding_box_offset): self.bounding_box = BoundingBoxWithUnknown( reader) if bounding_box_has_unknown else BoundingBox( reader) bone_count = mesh.pop("__bone_count") with reader.temp_offset(mesh.pop("__bone_offset")): self.bone_indices = list(reader.unpack(f"<{bone_count}i")) face_set_count = mesh.pop("__face_set_count") with reader.temp_offset(mesh.pop("__face_set_offset")): self._face_set_indices = list(reader.unpack(f"<{face_set_count}i")) vertex_count = mesh.pop("__vertex_buffer_count") with reader.temp_offset(mesh.pop("__vertex_buffer_offset")): self._vertex_buffer_indices = list( reader.unpack(f"<{vertex_count}i")) self.set(**mesh)
def get_instruction_args(reader: BinaryReader, category, index, first_arg_offset, event_args_size, emedf: dict): """Process instruction arguments (required and optional) from EMEVD binary.""" try: emedf_args_info = emedf[category, index]["args"] except KeyError: raise KeyError( f"Could not find instruction ({category}, {index}) in `Instruction.EMEDF`." ) previous_offset = reader.position if event_args_size == 0: return "", [] try: args_format = "@" + "".join(arg["internal_type"].get_fmt() for arg in emedf_args_info.values()) except KeyError: raise KeyError( f"Cannot find argument types for instruction {category}[{index:02d}] ({event_args_size} bytes)" ) # 's' arguments are actually four-byte offsets into the packed string data, though we will keep the 's' symbol. struct_args_format = args_format.replace("s", "I") required_args_size = struct.calcsize(struct_args_format) if required_args_size > event_args_size: raise ValueError( f"Documented size of minimum required args for instruction {category}" f"[{index}] is {required_args_size}, but size of args specified in EMEVD file is " f"only {event_args_size}.") reader.seek(first_arg_offset) args = reader.unpack(struct_args_format) # Additional arguments may appear for the instruction 2000[00], 'RunEvent'. These instructions are tightly packed # and are always aligned to 4. We read them here as unsigned integers and must actually parse the called event ID to # interpret them properly (done at `EMEVD` class level). extra_size = event_args_size - required_args_size opt_arg_count = extra_size // 4 if opt_arg_count == 0: reader.seek(previous_offset) return args_format[1:], list(args) elif (category, index) not in _OPTIONAL_ARGS_ALLOWED: raise ValueError( f"Extra arguments found for instruction {category}[{index}], which is not permitted. Arg types may be " f"wrong (too short) for this instruction.\n" f" required size = {required_args_size}\n" f" actual size = {event_args_size}") elif extra_size % 4 != 0: raise ValueError( f"Error interpreting instruction {category}[{index}]: optional argument " f"size is not a multiple of four bytes ({extra_size}).") opt_args = [reader.unpack_value("<I") for _ in range(opt_arg_count)] reader.seek(previous_offset) return args_format[1:] + "|" + "I" * (extra_size // 4), list(args) + opt_args
def unpack_from(cls, reader: BinaryReader): float_struct = cls() float_struct.unk0 = reader.unpack_value("i") length = reader.unpack_value("i") if length < 0 or length % 4: raise ValueError( f"Unexpected `FloatStruct` length: {length}. Expected a multiple of 4 (or 0)." ) float_struct.values = list(reader.unpack(f"{length // 4}f"))
def ENTRY_CLASS(cls, msb_reader: BinaryReader): """Detects the appropriate subclass of `MSBPart` to instantiate, and does so.""" entry_subtype_int = msb_reader.unpack("i", offset=cls.SUBTYPE_OFFSET, relative_offset=True)[0] try: entry_subtype = cls.ENTRY_SUBTYPE_ENUM(entry_subtype_int) except ValueError: raise MapError( f"Entry of type {cls.ENTRY_SUBTYPE_ENUM} has invalid subtype enum: {entry_subtype_int}" ) return cls.SUBTYPE_CLASSES[entry_subtype](msb_reader)
def __init__(self, data: bytes): reader = BinaryReader(data) reader.unpack_value("4s", asserted=b"DDS ") reader.unpack_value("i", asserted=0x7C) self.flags = reader.unpack_value("I") self.height = reader.unpack_value("i") self.width = reader.unpack_value("i") self.pitch_or_linear_size = reader.unpack_value("i") self.depth = reader.unpack_value("i") self.mipmap_count = reader.unpack_value("i") self.reserved_1 = reader.unpack("11i") # TODO: More here (see SoulsFormats excerpt below), but I care mainly about width/height right now. """
def get_instruction_args(reader: BinaryReader, category, index, first_arg_offset, event_args_size, format_dict): """Process instruction arguments (required and optional) from EMEVD binary.""" previous_offset = reader.position if event_args_size == 0: return "", [] try: args_format = "@" + format_dict[category][index] except KeyError: raise KeyError( f"Cannot find argument types for instruction {category}[{index:02d}]." ) # 's' arguments are actually four-byte offsets into the packed string data, though we will keep the 's' symbol. struct_args_format = args_format.replace("s", "I") required_args_size = struct.calcsize(struct_args_format) if required_args_size > event_args_size: raise ValueError( f"Documented size of minimum required args for instruction {category}" f"[{index}] is {required_args_size}, but size of args specified in EMEVD file is " f"only {event_args_size}.") reader.seek(first_arg_offset) args = reader.unpack(struct_args_format) # Additional arguments may appear for the instruction 2000[00], 'RunEvent'. These instructions are tightly packed # and are always aligned to 4. We read them here as unsigned integers and must actually parse the called event ID to # interpret them properly (done at `EMEVD` class level). extra_size = event_args_size - required_args_size opt_arg_count = extra_size // 4 if opt_arg_count == 0: reader.seek(previous_offset) return args_format[1:], list(args) elif extra_size % 4 != 0: raise ValueError( f"Error interpreting instruction {category}[{index}]: optional argument " f"size is not a multiple of four bytes ({extra_size}).") opt_args = [reader.unpack_value("<I") for _ in range(opt_arg_count)] reader.seek(previous_offset) return args_format[1:] + "|" + "I" * (extra_size // 4), list(args) + opt_args
def unpack(self, reader: BinaryReader, header_vertex_index_size: int, vertex_data_offset: int): face_set = reader.unpack_struct(self.STRUCT) vertex_index_size = face_set.pop("__vertex_index_size") if vertex_index_size == 0: vertex_index_size = header_vertex_index_size if vertex_index_size == 8: raise NotImplementedError( "Soulstruct cannot support edge-compressed FLVER face sets.") elif vertex_index_size in {16, 32}: vertex_indices_count = face_set.pop("__vertex_indices_count") vertex_indices_offset = face_set.pop("__vertex_indices_offset") with reader.temp_offset(vertex_data_offset + vertex_indices_offset): fmt = f"<{vertex_indices_count}{'H' if vertex_index_size == 16 else 'I'}" self.vertex_indices = list(reader.unpack(fmt)) else: raise ValueError( f"Unsupported face set index size: {vertex_index_size}") self.set(**face_set)
def unpack(self, reader: BinaryReader, **kwargs): self.byte_order = reader.byte_order = ">" if reader.unpack_value( "B", offset=44) == 255 else "<" version_info = reader.unpack("bbb", offset=45) self.flags1 = ParamFlags1(version_info[0]) self.flags2 = ParamFlags2(version_info[1]) self.paramdef_format_version = version_info[2] header_struct = self.GET_HEADER_STRUCT(self.flags1, self.byte_order) header = reader.unpack_struct(header_struct) try: self.param_type = header["param_type"] except KeyError: self.param_type = reader.unpack_string( offset=header["param_type_offset"], encoding="utf-8") self.paramdef_data_version = header["paramdef_data_version"] self.unknown = header["unknown"] # Row data offset in header not used. (It's an unsigned short, yet doesn't limit row count to 5461.) name_data_offset = header[ "name_data_offset"] # CANNOT BE TRUSTED IN VANILLA FILES! Off by +12 bytes. # Load row pointer data. row_struct = self.ROW_STRUCT_64 if self.flags1.LongDataOffset else self.ROW_STRUCT_32 row_pointers = reader.unpack_structs(row_struct, count=header["row_count"]) row_data_offset = reader.position # Reliable row data offset. # Row size is lazily determined. TODO: Unpack row data in sequence and associate with names separately. if len(row_pointers) == 0: return elif len(row_pointers) == 1: # NOTE: The only vanilla param in Dark Souls with one row is LEVELSYNC_PARAM_ST (Remastered only), # for which the row size is hard-coded here. Otherwise, we can trust the repacked offset from Soulstruct # (and SoulsFormats, etc.). if self.param_type == "LEVELSYNC_PARAM_ST": row_size = 220 else: row_size = name_data_offset - row_data_offset else: row_size = row_pointers[1]["data_offset"] - row_pointers[0][ "data_offset"] # Note that we no longer need to track reader offset. name_encoding = self.get_name_encoding() for row_struct in row_pointers: reader.seek(row_struct["data_offset"]) row_data = reader.read(row_size) if row_struct["name_offset"] != 0: try: name = reader.unpack_string( offset=row_struct["name_offset"], encoding=name_encoding, reset_old_offset=False, # no need to reset ) except UnicodeDecodeError as ex: if ex.object in self.undecodable_row_names: name = reader.unpack_bytes( offset=row_struct["name_offset"], reset_old_offset=False, # no need to reset ) else: raise except ValueError: reader.seek(row_struct["name_offset"]) _LOGGER.error( f"Error encountered while parsing row name string in {self.param_type}.\n" f" Header: {header}\n" f" Row Struct: {row_struct}\n" f" 30 chrs of name data: {' '.join(f'{{:02x}}'.format(x) for x in reader.read(30))}" ) raise else: name = "" self.rows[row_struct["id"]] = ParamRow(row_data, self.paramdef, name=name)
def read(self, reader: BinaryReader, layout: BufferLayout, uv_factor: float): self.uvs = [] self.tangents = [] self.colors = [] with reader.temp_offset(reader.position): self.raw = reader.read(layout.get_total_size()) for member in layout: not_implemented = False if member.semantic == LayoutSemantic.Position: if member.layout_type == LayoutType.Float3: self.position = Vector3(reader.unpack("<3f")) elif member.layout_type == LayoutType.Float4: self.position = Vector3(reader.unpack("<3f"))[:3] elif member.layout_type == LayoutType.EdgeCompressed: raise NotImplementedError( "Soulstruct cannot load FLVERs with edge-compressed vertex positions." ) else: not_implemented = True elif member.semantic == LayoutSemantic.BoneWeights: if member.layout_type == LayoutType.Byte4A: self.bone_weights = VertexBoneWeights( *[w / 127.0 for w in reader.unpack("<4b")]) elif member.layout_type == LayoutType.Byte4C: self.bone_weights = VertexBoneWeights( *[w / 255.0 for w in reader.unpack("<4B")]) elif member.layout_type in { LayoutType.UVPair, LayoutType.Short4ToFloat4A }: self.bone_weights = VertexBoneWeights( *[w / 32767.0 for w in reader.unpack("<4h")]) else: not_implemented = True elif member.semantic == LayoutSemantic.BoneIndices: if member.layout_type in { LayoutType.Byte4B, LayoutType.Byte4E }: self.bone_indices = VertexBoneIndices( *reader.unpack("<4B")) elif member.layout_type == LayoutType.ShortBoneIndices: self.bone_indices = VertexBoneIndices( *reader.unpack("<4h")) else: not_implemented = True elif member.semantic == LayoutSemantic.Normal: if member.layout_type == LayoutType.Float3: self.normal = Vector3(reader.unpack("<3f")) elif member.layout_type == LayoutType.Float4: self.normal = Vector3(reader.unpack("<3f")) float_normal_w = reader.unpack_value("<f") self.normal_w = int(float_normal_w) if self.normal_w != float_normal_w: raise ValueError( f"`normal_w` float was not a whole number.") elif member.layout_type in { LayoutType.Byte4A, LayoutType.Byte4B, LayoutType.Byte4C, LayoutType.Byte4E }: self.normal = Vector3([(x - 127) / 127.0 for x in reader.unpack("<3B")]) self.normal_w = reader.unpack_value("<B") elif member.layout_type == LayoutType.Short2toFloat2: self.normal_w = reader.unpack_value("<B") self.normal = Vector3( [x / 127.0 for x in reader.unpack("<3b")]) elif member.layout_type == LayoutType.Short4ToFloat4A: self.normal = Vector3( [x / 32767.0 for x in reader.unpack("<3h")]) self.normal_w = reader.unpack_value("<h") elif member.layout_type == LayoutType.Short4ToFloat4B: self.normal = Vector3([(x - 32767) / 32767.0 for x in reader.unpack("<3H")]) self.normal_w = reader.unpack_value("<h") else: not_implemented = True elif member.semantic == LayoutSemantic.UV: if member.layout_type == LayoutType.Float2: self.uvs.append(Vector3(*reader.unpack("<2f"), 0.0)) elif member.layout_type == LayoutType.Float3: self.uvs.append(Vector3(*reader.unpack("<3f"))) elif member.layout_type == LayoutType.Float4: self.uvs.append(Vector3(*reader.unpack("<2f"), 0.0)) self.uvs.append(Vector3(*reader.unpack("<2f"), 0.0)) elif member.layout_type in { LayoutType.Byte4A, LayoutType.Byte4B, LayoutType.Short2toFloat2, LayoutType.Byte4C, LayoutType.UV }: self.uvs.append( Vector3(*reader.unpack("<2h"), 0) / uv_factor) elif member.layout_type == LayoutType.UVPair: self.uvs.append( Vector3(*reader.unpack("<2h"), 0) / uv_factor) self.uvs.append( Vector3(*reader.unpack("<2h"), 0) / uv_factor) elif member.layout_type == LayoutType.Short4ToFloat4B: self.uvs.append(Vector3(*reader.unpack("<3h")) / uv_factor) if reader.unpack_value("<h") != 0: raise ValueError( "Expected zero short after reading UV | Short4ToFloat4B vertex member." ) else: not_implemented = True elif member.semantic == LayoutSemantic.Tangent: if member.layout_type == LayoutType.Float4: self.tangents.append(Vector4(*reader.unpack("<4f"))) elif member.layout_type in { LayoutType.Byte4A, LayoutType.Byte4B, LayoutType.Byte4C, LayoutType.Short4ToFloat4A, LayoutType.Byte4E, }: tangent = Vector4([(x - 127) / 127.0 for x in reader.unpack("<4B")]) self.tangents.append(tangent) else: not_implemented = True elif member.semantic == LayoutSemantic.Bitangent: if member.layout_type in { LayoutType.Byte4A, LayoutType.Byte4B, LayoutType.Byte4C, LayoutType.Byte4E }: self.bitangent = Vector4([(x - 127) / 127.0 for x in reader.unpack("<4B")]) else: not_implemented = True elif member.semantic == LayoutSemantic.VertexColor: if member.layout_type == LayoutType.Float4: self.colors.append(ColorRGBA(*reader.unpack("<4f"))) elif member.layout_type in { LayoutType.Byte4A, LayoutType.Byte4C }: # Convert byte channnels [0-255] to float channels [0-1]. self.colors.append( ColorRGBA(*[b / 255.0 for b in reader.unpack("<4B")])) else: not_implemented = True else: not_implemented = True if not_implemented: raise NotImplementedError( f"Unsupported vertex member semantic/type combination: " f"{member.semantic.name} | {member.layout_type.name}")