class StringTable(BinaryObject):
"""String table."""
_magic = b'_STR'
_reader = StructReader(
('4s', 'magic'), Padding(4),
('I', 'size'), Padding(4),
('I', 'num_strs'), Padding(4),
size = 0x18,
)
def _unpackFromData(self, data):
super()._unpackFromData(data)
self.strings = []
self._file.seek(self._file_offset + self._reader.size)
for i in range(self.num_strs):
offs = self._file.tell()
offs += (offs & 1) # pad to u16
self.strings.append(readStringWithLength(self._file, '<H', offs))
#log.debug('Str 0x%04X: "%s"', i, self.strings[-1])
return self
def validate(self):
super().validate()
return True
class FSHP(FresObject):
"""A FSHP in an FMDL."""
_magic = b'FSHP'
_reader = StructReader(
('4s', 'magic'),
('I', 'unk04'),
('I', 'unk08'),
('I', 'unk0C'),
StrOffs('name'),
Padding(4),
Offset64('fvtx_offset'), # => FVTX
Offset64('lod_offset'), # => LOD models
Offset64(
'fskl_idx_array_offs'
), # => 00030002 00050004 00070006 00090008 000B000A 000D000C 000F000E 00110010
Offset64('unk30'), # 0
Offset64('unk38'), # 0
# bounding box and bounding radius
Offset64('bbox_offset'), # => about 24 floats, or 8 Vec3s, or 6 Vec4s
Offset64(
'bradius_offset'
), # => => 3F03ADA8 3EFC1658 00000000 00000D14 00000000 00000000 00000000 00000000
Offset64('unk50'),
('I', 'flags'),
('H', 'index'),
('H', 'fmat_idx'),
('H', 'single_bind'),
('H', 'fvtx_idx'),
('H', 'skin_bone_idx_cnt'),
('B', 'vtx_skin_cnt'),
('B', 'lod_cnt'),
('I', 'vis_group_cnt'),
('H', 'fskl_array_cnt'),
Padding(2),
size=0x70,
)
def readFromFRES(self, fres, offset=None, reader=None):
"""Read the FSHP from given FRES."""
super().readFromFRES(fres, offset, reader)
#log.debug("FSHP name='%s'", self.name)
self.dumpToDebugLog()
#self.dumpOffsets()
self.fvtx = FVTX().readFromFRES(fres, self.fvtx_offset)
self.lods = []
for i in range(self.lod_cnt):
model = LODModel().readFromFRES(
fres, self.lod_offset + (i * LODModel._reader.size))
self.lods.append(model)
#self.lods = [self.lods[1]] # XXX DEBUG only keep one model
return self
def validate(self):
super().validate()
return True
class ShaderAssign(FresObject):
_reader = StructReader(
StrOffs('name'),
Padding(4),
StrOffs('name2'),
Padding(4),
Offset64('vtx_attr_names'), # -> offsets of attr names
Offset64('vtx_attr_dict'),
Offset64('tex_attr_names'),
Offset64('tex_attr_dict'),
Offset64('mat_param_vals'), # names from dict
Offset64('mat_param_dict'),
Padding(4),
('B', 'num_vtx_attrs'),
('B', 'num_tex_attrs'),
('H', 'num_mat_params'),
)
class BNTX(BinaryObject):
"""BNTX texture pack."""
_magic = b'BNTX'
_reader = StructReader(
('4s', 'magic'),
Padding(4),
('I', 'data_len'),
('H', 'byte_order'), # FFFE or FEFF
('H', 'version'),
StrOffs('name'),
Padding(2),
('H', 'strings_offs'), # relative to start of BNTX
Offset('reloc_offs'),
('I', 'file_size'),
size=0x20,
)
def _unpackFromData(self, data):
super()._unpackFromData(data)
self.name = readString(self._file, self.name)
self.strings = StringTable().readFromFile(
self._file, self._file_offset + self.strings_offs)
self.nx = NX().readFromFile(self._file,
self._file_offset + self._reader.size)
#self.nx.dumpToDebugLog()
self.textures = []
for i in range(self.nx.num_textures):
offs = self._file.read('Q', self.nx.info_ptrs_offset + (i * 8))
brti = BRTI().readFromFile(self._file, offs)
log.debug("Tex %d = %s", i, brti)
self.textures.append(brti)
return self
def validate(self):
super().validate()
return True
def __str__(self):
return "<BNTX '%s' at 0x%X>" % (self.name, id(self))
class EmbeddedFile(FresObject):
"""Generic file embedded in FRES archive."""
_reader = StructReader(
Offset64('data_offset'),
('I', 'size'),
Padding(4),
size=0x10,
)
def readFromFRES(self, fres, offset=None, reader=None):
"""Read the archive from given FRES."""
super().readFromFRES(fres, offset, reader)
#self.dumpToDebugLog()
#self.dumpOffsets()
return self
def toData(self):
return self.fres.file.read(self.size, self.data_offset)
class Dict(BinaryObject):
"""Dictionary of names."""
_reader = StructReader(
('i', 'unk00'), # always 0?
('i', 'numItems'),
size=8,
)
_itemReader = StructReader(
('i', 'search'), # mostly increases, not always by 1
('h', 'left'), # usually -1 for first item
('h', 'right'), # usually 1 for first item
StrOffs('nameoffs'),
Padding(4),
size=0x10,
)
def _unpackFromData(self, data):
super()._unpackFromData(data)
self.items = []
for i in range(self.numItems + 1):
self._file.seek(self._file_offset + self._reader.size +
(i * self._itemReader.size))
item = self._itemReader.unpackFromFile(self._file)
if item['nameoffs'] == 0: break
item['name'] = readStringWithLength(self._file, '<H',
item['nameoffs'])
self.items.append(item)
return self
def dumpToDebugLog(self):
"""Dump to debug log."""
log.debug("Dict with %d items; unk00 = %d", self.numItems, self.unk00)
for i, item in enumerate(self.items):
log.debug('%4d: %4d, %4d, %4d, "%s"', i, item['search'],
item['left'], item['right'], item['name'])
def validate(self):
super().validate()
return True
class Attribute(FresObject):
"""An attribute in a FRES."""
_reader = StructReader(
StrOffs('name'),
('I', 'unk04'),
('H', 'format'),
Padding(2),
('H', 'buf_offs'),
('H', 'buf_idx'),
size=0x10,
)
def readFromFRES(self, fres, offset=None, reader=None):
"""Read the attribute from given FRES."""
super().readFromFRES(fres, offset, reader)
log.debug("Attr name='%s' fmt=%04X offs=%d idx=%d unk=%d", self.name,
self.format, self.buf_offs, self.buf_idx, self.unk04)
#self.dumpToDebugLog()
self.fvtx = None # to be filled in by the FVTX that reads it
return self
def validate(self):
super().validate()
return True
class BRTI(BinaryObject):
"""A BRTI in a BNTX."""
class ChannelType(Enum):
Zero = 0
One = 1
Red = 2
Green = 3
Blue = 4
Alpha = 5
class TextureType(Enum):
Image1D = 0
Image2D = 1
Image3D = 2
Cube = 3
CubeFar = 8
class TextureDataType(Enum):
UNorm = 1
SNorm = 2
UInt = 3
SInt = 4
Single = 5
SRGB = 6
UHalf = 10
defaultFileExt = 'png'
_magic = b'BRTI'
_reader = StructReader(
('4s', 'magic'),
('I', 'length'),
('Q', 'length2'),
('B', 'flags'),
('B', 'dimensions'),
('H', 'tile_mode'),
('H', 'swizzle_size'),
('H', 'mipmap_cnt'),
('H', 'multisample_cnt'),
('H', 'reserved1A'),
('B', 'fmt_dtype', lambda v: BRTI.TextureDataType(v)),
('B', 'fmt_type', lambda v: TextureFormat.get(v)()),
Padding(2),
('I', 'access_flags'),
('i', 'width'),
('i', 'height'),
('i', 'depth'),
('i', 'array_cnt'),
('i', 'block_height', lambda v: 2**v),
('H', 'unk38'),
('H', 'unk3A'),
('i', 'unk3C'),
('i', 'unk40'),
('i', 'unk44'),
('i', 'unk48'),
('i', 'unk4C'),
('i', 'data_len'),
('i', 'alignment'),
('4B', 'channel_types', lambda v: tuple(map(BRTI.ChannelType, v))),
('i', 'tex_type'),
StrOffs('name'),
Padding(4),
Offset64('parent_offset'),
Offset64('ptrs_offset'),
)
def _unpackFromData(self, data):
super()._unpackFromData(data)
self.name = readStringWithLength(self._file, '<H', self.name)
#self.dumpToDebugLog()
self.swizzle = BlockLinearSwizzle(self.width,
self.fmt_type.bytesPerPixel,
self.block_height)
#self.swizzle = Swizzle(self.width,
# self.fmt_type.bytesPerPixel,
# self.block_height)
self._readMipmaps()
self._readData()
log.debug("Texture '%s' size %dx%dx%d, len=%d: %s", self.name,
self.width, self.height, self.depth, self.data_len,
' '.join(map(lambda b: '%02X' % b, self.data[0:16])))
return self
def _readMipmaps(self):
self.mipOffsets = []
for i in range(self.mipmap_cnt):
offs = self.ptrs_offset + (i * 8)
entry = self._file.read('I', offs) #- base
self.mipOffsets.append(entry)
log.debug("mipmap offsets: %s",
list(map(lambda o: '%08X' % o, self.mipOffsets)))
def _readData(self):
base = self._file.read('Q', self.ptrs_offset)
log.debug("Data at 0x%X => 0x%X", self.ptrs_offset, base)
self.data = self._file.read(self.data_len, base)
def decode(self):
self.pixels, self.depth = self.fmt_type.decode(self)
return self.pixels
def toData(self):
self.decode()
tempFile = tempfile.SpooledTemporaryFile()
png = PNG(width=self.width,
height=self.height,
pixels=self.pixels,
bpp=self.depth)
png.writeToFile(tempFile)
tempFile.seek(0)
return tempFile.read()
def validate(self):
super().validate()
return True
class FSKL(FresObject):
"""FSKL object header."""
# offsets in this struct are relative to the beginning of
# the FRES file.
# I'm assuming they're 64-bit.
_magic = b'FSKL'
_reader = StructReader(
('4s', 'magic'),
('I', 'size'),
('I', 'size2'),
Padding(4),
Offset64('bone_idx_group_offs'),
Offset64('bone_array_offs'),
Offset64('smooth_idx_offs'),
Offset64('smooth_mtx_offs'),
Offset64('unk30'),
Flags(
'flags',
{
#'SCALE_NONE': 0x00000000, # no scaling
'SCALE_STD': 0x00000100, # standard scaling
'SCALE_MAYA': 0x00000200, # Respects Maya's segment scale
# compensation which offsets child bones rather than
# scaling them with the parent.
'SCALE_SOFTIMAGE': 0x00000300, # Respects the scaling method
# of Softimage.
'EULER': 0x00001000, # euler rotn, not quaternion
}),
('H', 'num_bones'),
('H', 'num_smooth_idxs'),
('H', 'num_rigid_idxs'),
('H', 'num_extra'),
('I', 'unk44'),
size=0x48,
)
def readFromFRES(self, fres, offset=None, reader=None):
"""Read the skeleton from given FRES."""
super().readFromFRES(fres, offset, reader)
self.dumpToDebugLog()
self.dumpOffsets()
scaleModes = ('none', 'standard', 'maya', 'softimage')
log.debug(
"Skeleton contains %d bones, %d smooth idxs, %d rigid idxs, %d extras; scale mode=%s, rotation=%s; smooth_mtx_offs=0x%X",
self.num_bones, self.num_smooth_idxs, self.num_rigid_idxs,
self.num_extra, scaleModes[(self.flags['_raw'] >> 8) & 3],
'euler' if self.flags['EULER'] else 'quaternion',
self.smooth_mtx_offs)
self._readSmoothIdxs(fres)
self._readSmoothMtxs(fres)
self._readBones(fres)
return self
def _readBones(self, fres):
self.bones = []
self.bonesByName = {}
self.boneIdxGroups = []
offs = self.bone_array_offs
for i in range(self.num_bones):
b = Bone().readFromFRES(fres, offs)
self.bones.append(b)
if b.name in self.bonesByName:
log.warn("Duplicate bone name '%s'", b.name)
self.bonesByName[b.name] = b
offs += Bone._reader.size
# set parents
for bone in self.bones:
bone.fskl = self
if bone.parent_idx >= len(self.bones):
log.error("Bone %s has invalid parent_idx %d (max is %d)",
bone.name, bone.parent_idx, len(self.bones))
bone.parent = None
elif bone.parent_idx >= 0:
bone.parent = self.bones[bone.parent_idx]
else:
bone.parent = None
log.debug(
"Skeleton: |Final Position |Final Rotation|Raw Position |Raw Rotation"
)
self.bones[0].printHeirarchy(self)
#log.debug("Final bone transforms:")
#for bone in self.bones:
# log.debug("%s\n%s", bone.name, bone.computeTransform())
self.boneIdxGroups = Dict().readFromFile(self.fres.file,
self.bone_idx_group_offs)
def _readSmoothIdxs(self, fres):
self.smooth_idxs = fres.read('h',
pos=self.smooth_idx_offs,
count=self.num_smooth_idxs)
log.debug("Smooth idxs: %s", self.smooth_idxs)
def _readSmoothMtxs(self, fres):
"""Read the smooth matrices."""
self.smooth_mtxs = []
for i in range(max(self.smooth_idxs)):
mtx = fres.read('3f',
count=4,
pos=self.smooth_mtx_offs + (i * 16 * 3))
# warn about invalid values
for y in range(4):
for x in range(3):
n = mtx[y][x]
if math.isnan(n) or math.isinf(n):
log.warning(
"Skeleton smooth mtx %d element [%d,%d] is %s", i,
x, y, n)
# replace all invalid values with zeros
flt = lambda e: \
0 if (math.isnan(e) or math.isinf(e)) else e
mtx = list(map(lambda row: list(map(flt, row)), mtx))
#mtx[3][3] = 1 # debug
mtx = Matrix(*mtx)
# transpose
#m = [[0,0,0,0], [0,0,0,0], [0,0,0,0], [0,0,0,0]]
#for y in range(4):
# for x in range(4):
# m[x][y] = mtx[y][x]
#mtx = m
# log values to debug
#log.debug("Inverse mtx %d:", i)
#for y in range(4):
# log.debug(" %s", ' '.join(map(
# lambda v: '%+3.2f' % v, mtx[y])))
self.smooth_mtxs.append(mtx)
def validate(self):
#for field in self._reader.fields.values():
# val = getattr(self, field['name'])
# if type(val) is int:
# log.debug("FMDL[%04X] %16s = 0x%08X", field['offset'],
# field['name'], val)
# else:
# log.debug("FMDL[%04X] %16s = %s", field['offset'],
# field['name'], val)
super().validate()
return True
class FMDL(FresObject):
"""FMDL object header."""
#defaultFileExt = 'x3d'
defaultFileExt = 'dae'
# offsets in this struct are relative to the beginning of
# the FRES file.
# I'm assuming they're 64-bit since most are a 32-bit offset
# followed by 4 zero bytes.
_magic = b'FMDL'
_reader = StructReader(
('4s', 'magic'),
('I', 'size'),
('I', 'size2'),
Padding(4),
StrOffs('name'),
Padding(4),
Offset64('str_tab_end'),
Offset64('fskl_offset'),
Offset64('fvtx_offset'),
Offset64('fshp_offset'),
Offset64('fshp_dict_offset'),
Offset64('fmat_offset'),
Offset64('fmat_dict_offset'),
Offset64('udata_offset'),
Offset64('unk60'),
Offset64('unk68'), # probably dict for unk60
('H', 'fvtx_count'),
('H', 'fshp_count'),
('H', 'fmat_count'),
('H', 'udata_count'),
('H', 'total_vtxs'),
Padding(6),
size = 0x78,
)
def readFromFRES(self, fres, offset=None, reader=None):
"""Read the archive from given FRES."""
super().readFromFRES(fres, offset, reader)
log.debug("FMDL name: '%s'", self.name)
self.dumpToDebugLog()
#self.dumpOffsets()
log.debug("FMDL '%s' contains %d skeletons, %d FVTXs, %d FSHPs, %d FMATs, %d udatas, total %d vertices",
self.name,
1 if self.fskl_offset > 0 else 0, # can this ever be 0?
self.fvtx_count, self.fshp_count, self.fmat_count,
self.udata_count, self.total_vtxs)
# read skeleton
self.skeleton = FSKL().readFromFRES(fres, self.fskl_offset)
# read vertex objects
self.fvtxs = []
for i in range(self.fvtx_count):
vtx = FVTX().readFromFRES(fres,
self.fvtx_offset + (i*FVTX._reader.size))
self.fvtxs.append(vtx)
# read shapes
self.fshps = []
for i in range(self.fshp_count):
self.fshps.append(FSHP().readFromFRES(fres,
self.fshp_offset + (i*FSHP._reader.size)))
# read materials
self.fmats = []
for i in range(self.fmat_count):
self.fmats.append(FMAT().readFromFRES(fres,
self.fmat_offset + (i*FMAT._reader.size)))
#self.fshps = [self.fshps[1]] # XXX DEBUG only keep one model
return self
def validate(self):
super().validate()
return True
def toData(self):
"""Export model to X3D file."""
#writer = X3DWriter(self)
writer = ColladaWriter()
for i, fmat in enumerate(self.fmats):
writer.addFMAT(fmat)
for i, fvtx in enumerate(self.fvtxs):
writer.addFVTX(fvtx, name=self.fshps[i].name)
writer.addFSHP(self.fshps[i]) # XXX this is weird
writer.addFSKL(self.skeleton)
writer.addScene()
return writer.toXML().tostring(pretty_print=True)
class FMAT(FresObject):
"""A FMAT in an FMDL."""
_magic = b'FMAT'
_reader = StructReader(
('4s', 'magic'),
('I', 'size'),
('I', 'size2'),
Padding(4),
StrOffs('name'),
Padding(4),
Offset64('render_param_offs'),
Offset64('render_param_dict_offs'),
Offset64('shader_assign_offs'), # -> name offsets
Offset64('unk30_offs'),
Offset64('tex_ref_array_offs'),
Offset64('unk40_offs'),
Offset64('sampler_list_offs'),
Offset64('sampler_dict_offs'),
Offset64('shader_param_array_offs'),
Offset64('shader_param_dict_offs'),
Offset64('shader_param_data_offs'),
Offset64('user_data_offs'),
Offset64('user_data_dict_offs'),
Offset64('volatile_flag_offs'),
Offset64('user_offs'),
Offset64('sampler_slot_offs'),
Offset64('tex_slot_offs'),
('I', 'mat_flags'),
('H', 'section_idx'),
('H', 'render_param_cnt'),
('B', 'tex_ref_cnt'),
('B', 'sampler_cnt'),
('H', 'shader_param_cnt'),
('H', 'shader_param_data_size'),
('H', 'raw_param_data_size'),
('H', 'user_data_cnt'),
Padding(2),
('I', 'unkB4'),
size=0xB8,
)
def readFromFRES(self, fres, offset=None, reader=None):
"""Read the FMAT from given FRES."""
super().readFromFRES(fres, offset, reader)
log.debug("FMAT name='%s'", self.name)
self.dumpToDebugLog()
self.dumpOffsets()
self._readDicts()
self._readRenderParams()
self._readShaderParams()
self._readTextureList()
self._readSamplerList()
self._readShaderAssign()
return self
def _readDicts(self):
dicts = ('render_param', 'sampler', 'shader_param', 'user_data')
for name in dicts:
offs = getattr(self, name + '_dict_offs')
if offs:
#d = IndexGroup().readFromFile(self.fres.file, offs)
#log.debug("FMAT %s dict:\n%s", name, d.dump())
data = self._readDict(offs, name)
else:
data = None
setattr(self, name + '_dict', data)
def _readDict(self, offs, name):
d = Dict().readFromFile(self.fres.file, offs)
log.debug("FMAT dict %s:", name)
d.dumpToDebugLog()
return d
def _readTextureList(self):
self.textures = []
log.debug("Texture list:")
for i in range(self.tex_ref_cnt):
name = self.fres.readStrPtr(self.tex_ref_array_offs + (i * 8))
slot = self.fres.read('q', self.tex_slot_offs + (i * 8))
log.debug("%3d (%2d): %s", i, slot, name)
self.textures.append({'name': name, 'slot': slot})
def _readSamplerList(self):
self.samplers = []
log.debug("Sampler list:")
for i in range(self.sampler_cnt):
data = self.fres.readHexWords(8, self.sampler_list_offs + (i * 32))
slot = self.fres.read('q', self.sampler_slot_offs + (i * 8))
log.debug("%3d (%2d): %s", i, slot, data)
self.samplers.append({'slot': slot, 'data': data})
# XXX no idea what to do with this data
def _readRenderParams(self):
self.renderParams = {}
types = ('?', 'float', 'str')
for i in range(self.render_param_cnt):
name, offs, cnt, typ, pad = self.fres.read(
'QQHHI', self.render_param_offs + (i * 24))
name = self.fres.readStr(name)
if pad != 0:
log.warning("Render info '%s' padding=0x%X", name, pad)
try:
typeName = types[typ]
except IndexError:
typeName = '0x%X' % typ
param = {
'name': name,
'count': cnt,
'type': types[typ],
'vals': [],
}
for j in range(cnt):
if typ == 0: val = self.fres.readHex(8, offs)
elif typ == 1: val = self.fres.read('f', offs)
elif typ == 2: val = self.fres.readStrPtr(offs)
else:
log.warning("Render param '%s' unknown type 0x%X", name,
typ)
val = '<unknown>'
param['vals'].append(val)
#log.debug("Render param: %-5s[%d] %-32s: %s",
# typeName, cnt, name, ', '.join(map(str, param['vals'])))
if name in self.renderParams:
log.warning("Duplicate render param '%s'", name)
self.renderParams[name] = param
def _readShaderParams(self):
self.shaderParams = {}
#log.debug("Shader params:")
array_offs = self.shader_param_array_offs
data_offs = self.shader_param_data_offs
for i in range(self.shader_param_cnt):
# unk0: always 0; unk14: always -1
# idx0, idx1: both always == i
unk0, name, type, size, offset, unk14, idx0, idx1 = \
self.fres.read('QQBBHiHH', array_offs + (i*32))
name = self.fres.readStr(name)
type = shaderParamTypes[type]
if unk0: log.debug("Shader param '%s' unk0=0x%X", name, unk0)
if unk14 != -1:
log.debug("Shader param '%s' unk14=%d", name, unk14)
if idx0 != i or idx1 != i:
log.debug("Shader param '%s' idxs=%d, %d (expected %d)", name,
idx0, idx1, i)
data = self.fres.read(size, data_offs + offset)
data = struct.unpack(type['fmt'], data)
#log.debug("%-38s %-5s %s", name, type['name'],
# type['outfmt'] % data)
if name in self.shaderParams:
log.warning("Duplicate shader param '%s'", name)
self.shaderParams[name] = {
'name': name,
'type': type,
'size': size,
'offset': offset,
'idxs': (idx0, idx1),
'unk00': unk0,
'unk14': unk14,
'data': data,
}
def _readShaderAssign(self):
assign = ShaderAssign()
assign.readFromFRES(self.fres, self.shader_assign_offs)
self.shader_assign = assign
log.debug("shader assign: %d vtx attrs, %d tex attrs, %d mat params",
assign.num_vtx_attrs, assign.num_tex_attrs,
assign.num_mat_params)
self.vtxAttrs = []
for i in range(assign.num_vtx_attrs):
name = self.fres.readStrPtr(assign.vtx_attr_names + (i * 8))
log.debug("vtx attr %d: '%s'", i, name)
self.vtxAttrs.append(name)
self.texAttrs = []
for i in range(assign.num_tex_attrs):
name = self.fres.readStrPtr(assign.tex_attr_names + (i * 8))
log.debug("tex attr %d: '%s'", i, name)
self.texAttrs.append(name)
self.mat_param_dict = self._readDict(assign.mat_param_dict,
"mat_params")
self.mat_params = {}
#log.debug("material params:")
for i in range(assign.num_mat_params):
name = self.mat_param_dict.items[i + 1]['name']
val = self.fres.readStrPtr(assign.mat_param_vals + (i * 8))
#log.debug("%-40s: %s", name, val)
if name in self.mat_params:
log.warning("duplicate mat_param '%s'", name)
if name != '':
self.mat_params[name] = val
def validate(self):
super().validate()
return True