def _read_static_props_models(static_lump: BytesIO) -> Iterator[str]:
"""Read the static prop dictionary from the lump."""
[dict_num] = struct_read('<i', static_lump)
for _ in range(dict_num):
[padded_name] = struct_read('<128s', static_lump)
# Strip null chars off the end, and convert to a str.
yield padded_name.rstrip(b'\x00').decode('ascii')
def _parse_phy(self, f: BinaryIO, filename: str) -> None:
"""Parse the physics data file, if present.
"""
[
size,
header_id,
solid_count,
checksum,
] = ST_PHY_HEADER.unpack(f.read(ST_PHY_HEADER.size))
f.read(size - ST_PHY_HEADER.size) # If the header is larger ever.
for solid in range(solid_count):
[solid_size] = struct_read('i', f)
f.read(solid_size) # Skip the header.
self.phys_keyvalues = Property.parse(
read_nullstr(f),
filename + ":keyvalues",
allow_escapes=False,
single_line=True,
)
def static_props(self) -> Iterator['StaticProp']:
"""Read in the Static Props lump."""
# The version of the static prop format - different features.
try:
version = self.game_lumps[b'sprp'].version
except KeyError:
raise ValueError('No static prop lump!') from None
if version > 11:
raise ValueError('Unknown version ({})!'.format(version))
if version < 4:
# Predates HL2...
raise ValueError('Static prop version {} is too old!')
static_lump = BytesIO(self.game_lumps[b'sprp'].data)
# Array of model filenames.
model_dict = list(self._read_static_props_models(static_lump))
[visleaf_count] = struct_read('<i', static_lump)
visleaf_list = list(struct_read('H' * visleaf_count, static_lump))
[prop_count] = struct_read('<i', static_lump)
for i in range(prop_count):
origin = Vec(struct_read('fff', static_lump))
angles = Vec(struct_read('fff', static_lump))
[model_ind] = struct_read('<H', static_lump)
(
first_leaf,
leaf_count,
solidity,
flags,
skin,
min_fade,
max_fade,
) = struct_read('<HHBBiff', static_lump)
model_name = model_dict[model_ind]
visleafs = visleaf_list[first_leaf:first_leaf + leaf_count]
lighting_origin = Vec(struct_read('<fff', static_lump))
if version >= 5:
fade_scale = struct_read('<f', static_lump)[0]
else:
fade_scale = 1 # default
if version in (6, 7):
min_dx_level, max_dx_level = struct_read('<HH', static_lump)
else:
# Replaced by GPU & CPU in later versions.
min_dx_level = max_dx_level = 0 # None
if version >= 8:
(
min_cpu_level,
max_cpu_level,
min_gpu_level,
max_gpu_level,
) = struct_read('BBBB', static_lump)
else:
# None
min_cpu_level = max_cpu_level = 0
min_gpu_level = max_gpu_level = 0
if version >= 7:
r, g, b, renderfx = struct_read('BBBB', static_lump)
# Alpha isn't used.
tint = Vec(r, g, b)
else:
# No tint.
tint = Vec(255, 255, 255)
renderfx = 255
if version >= 11:
# Unknown data, though it's float-like.
unknown_1 = struct_read('<i', static_lump)
if version >= 10:
# Extra flags, post-CSGO.
flags |= struct_read('<I', static_lump)[0] << 8
flags = StaticPropFlags(flags)
scaling = 1.0
disable_on_xbox = False
if version >= 11:
# XBox support was removed. Instead this is the scaling factor.
[scaling] = struct_read("<f", static_lump)
elif version >= 9:
# The single boolean byte also produces 3 pad bytes.
[disable_on_xbox] = struct_read('<?xxx', static_lump)
yield StaticProp(
model_name,
origin,
angles,
scaling,
visleafs,
solidity,
flags,
skin,
min_fade,
max_fade,
lighting_origin,
fade_scale,
min_dx_level,
max_dx_level,
min_cpu_level,
max_cpu_level,
min_gpu_level,
max_gpu_level,
tint,
renderfx,
disable_on_xbox,
)
def read(self) -> None:
"""Load all data."""
self.lumps.clear()
self.game_lumps.clear()
with open(self.filename, mode='br') as file:
# BSP files start with 'VBSP', then a version number.
magic_name, bsp_version = struct_read(HEADER_1, file)
assert magic_name == BSP_MAGIC, 'Not a BSP file!'
if self.version is None:
try:
self.version = VERSIONS(bsp_version)
except ValueError:
self.version = bsp_version
else:
assert bsp_version == self.version, 'Different BSP version!'
lump_offsets = {}
# Read the index describing each BSP lump.
for index in range(LUMP_COUNT):
offset, length, version, ident = struct_read(HEADER_LUMP, file)
lump_id = BSP_LUMPS(index)
self.lumps[lump_id] = Lump(
lump_id,
version,
ident,
)
lump_offsets[lump_id] = offset, length
[self.map_revision] = struct_read(HEADER_2, file)
for lump in self.lumps.values():
# Now read in each lump.
offset, length = lump_offsets[lump.type]
file.seek(offset)
lump.data = file.read(length)
game_lump = self.lumps[BSP_LUMPS.GAME_LUMP]
self.game_lumps.clear()
[lump_count] = struct.unpack_from('<i', game_lump.data)
lump_offset = 4
for _ in range(lump_count):
(
game_lump_id,
flags,
glump_version,
file_off,
file_len,
) = GameLump.ST.unpack_from(
game_lump.data,
lump_offset) # type: bytes, int, int, int, int
lump_offset += GameLump.ST.size
file.seek(file_off)
# The lump ID is backward..
game_lump_id = game_lump_id[::-1]
self.game_lumps[game_lump_id] = GameLump(
game_lump_id,
flags,
glump_version,
file.read(file_len),
)
# This is not valid any longer.
game_lump.data = b''
def load_dirfile(self) -> None:
"""Read in the directory file to get all filenames.
This erases all changes in the file.
"""
if self.mode is OpenModes.WRITE:
# Erase the directory file, we ignore current contents.
open(self.path, 'wb').close()
self.version = 1
return
try:
dirfile = open(self.path, 'rb')
except FileNotFoundError:
if self.mode is OpenModes.APPEND:
# No directory file - generate a blank file.
open(self.path, 'wb').close()
self.version = 1
return
else:
raise # In read mode, don't overwrite and error when reading.
with dirfile:
vpk_sig, version, tree_length = struct_read('<III', dirfile)
if vpk_sig != VPK_SIG:
raise ValueError('Bad VPK directory signature!')
if version not in (1, 2):
raise ValueError("Bad VPK version {}!".format(self.version))
self.version = version
if version >= 2:
(
data_size,
ext_md5_size,
dir_md5_size,
sig_size,
) = struct_read('<4I', dirfile)
self.header_len = dirfile.tell() + tree_length
self._fileinfo.clear()
# Read directory contents
# These are in a tree of extension, directory, file. '' terminates a part.
for ext in iter_nullstr(dirfile):
ext_dict = self._fileinfo.setdefault(ext, {})
for directory in iter_nullstr(dirfile):
dir_dict = ext_dict.setdefault(directory, {})
for file in iter_nullstr(dirfile):
crc, index_len, arch_ind, offset, arch_len, end = struct_read(
'<IHHIIH', dirfile)
if arch_ind == DIR_ARCH_INDEX:
arch_ind = None
if arch_len == 0:
offset = 0
if end != 0xffff:
raise Exception(
'"{}" has bad terminator! {}'.format(
_join_file_parts(directory, file, ext),
(crc, index_len, arch_ind, offset,
arch_len, end),
))
dir_dict[file] = FileInfo(
self,
directory,
file,
ext,
crc=crc,
offset=offset,
start_data=dirfile.read(index_len),
arch_len=arch_len,
arch_index=arch_ind,
)
# 1 for the ending b'' section
if dirfile.tell() + 1 == self.header_len:
dirfile.read(1) # Skip null byte.
break
self.footer_data = dirfile.read()
def _cull_skins_table(self, f: BinaryIO, body_count: int) -> None:
"""Fix the table of used skins to correspond to those actually used.
StudioMDL is rather messy, and adds many extra columns that are not used
on the actual model.
We're following mstudiobodyparts_t -> mstudiomodel_t -> mstudiomesh_t -> material.
"""
used_inds = set()
# Iterate through bodygroups.
for body_ind in range(body_count):
body_start = f.tell()
(
body_name_off, # Offset to find the bodygroup name
model_count, # Number of models in this group
base, # Unknown
model_off,
) = struct_read('iiii', f)
body_end = f.tell()
f.seek(body_start + model_off)
for model_ind in range(model_count):
model_start = f.tell()
(
mdl_name,
mdl_type,
bound_radius,
mesh_count,
mesh_off,
num_verts,
vert_off,
tangent_off,
attach_count,
attach_ind,
eyeball_count,
eyeball_ind,
# Two void* pointers,
# 32 empty bytes
) = struct_read('64s i f 9i 8x 32x', f)
model_end = f.tell()
f.seek(model_start + mesh_off)
for mesh_ind in range(mesh_count):
(
material,
mesh_model_ind,
mesh_vert_count,
mesh_vert_off,
mesh_flex_count,
mesh_flex_ind,
mesh_mat_type,
mesh_mat_param,
mesh_id,
mesh_cent_x,
mesh_cent_y,
mesh_cent_z,
# Void pointer
# Array of LOD vertex counts ints, 8 of them
# 8 unused int spaces.
) = struct_read('9i 3f 4x 32x 32x', f)
used_inds.add(material)
f.seek(model_end)
f.seek(body_end)
for skin_ind, tex in enumerate(self.skins):
self.skins[skin_ind] = [tex[i] for i in used_inds]
def _read_sequences(f: BinaryIO, count: int) -> List[MDLSequence]:
"""Split this off to decrease stack in main parse method."""
sequences = [None] * count # type: List[MDLSequence]
for i in range(count):
start_pos = f.tell()
(
base_ptr,
label_pos,
act_name_pos,
flags,
_, # Seems to be a pointer.
act_weight,
event_count,
event_pos,
) = struct_read('8i', f)
bbox_min = str_readvec(f)
bbox_max = str_readvec(f)
# Skip 20 ints, 9 floats to get to keyvalues = 29*4 bytes
# Then 8 unused ints.
(
keyvalue_pos,
keyvalue_size,
) = struct_read('116xii32x', f)
end_pos = f.tell()
f.seek(start_pos + event_pos)
events = [None] * event_count # type: List[SeqEvent]
for j in range(event_count):
event_start = f.tell()
(
event_cycle,
event_index,
event_flags,
event_options,
event_nameloc,
) = struct_read('fii64si', f)
event_end = f.tell()
# There are two event systems.
if event_flags == 1 << 10:
# New system, name in the file.
event_name = read_nullstr(f, event_start + event_nameloc)
if event_name.isdigit():
try:
event_type = ANIM_EVENT_BY_INDEX[int(event_name)]
except KeyError:
raise ValueError('Unknown event index!')
else:
try:
event_type = ANIM_EVENT_BY_NAME[event_name]
except KeyError:
# NPC-specific events, declared dynamically.
event_type = event_name
else:
# Old system, index.
try:
event_type = ANIM_EVENT_BY_INDEX[event_index]
except KeyError:
# raise ValueError('Unknown event index!')
print('Unknown: ', event_index,
event_options.rstrip(b'\0'))
continue
f.seek(event_end)
events[j] = SeqEvent(
type=event_type,
cycle=event_cycle,
options=event_options.rstrip(b'\0').decode('ascii'))
if keyvalue_size:
keyvalues = read_nullstr(f, start_pos + keyvalue_pos)
else:
keyvalues = ''
sequences[i] = MDLSequence(
label=read_nullstr(f, start_pos + label_pos),
act_name=read_nullstr(f, start_pos + act_name_pos),
flags=flags,
act_weight=act_weight,
events=events,
bbox_min=bbox_min,
bbox_max=bbox_max,
keyvalues=keyvalues,
)
f.seek(end_pos)
return sequences
def _load(self, f: BinaryIO) -> None:
"""Read data from the MDL file."""
assert f.tell() == 0, "Doesn't begin at start?"
if f.read(4) != b'IDST':
raise ValueError('Not a model!')
(
self.version,
self.checksum,
name,
file_len,
) = struct_read('i 4s 64s i', f)
if not 44 <= self.version <= 49:
raise ValueError('Unknown MDL version {}!'.format(self.version))
self.name = name.rstrip(b'\0').decode('ascii')
self.eye_pos = str_readvec(f)
self.illum_pos = str_readvec(f)
# Approx dimensions
self.hull_min = str_readvec(f)
self.hull_max = str_readvec(f)
self.view_min = str_readvec(f)
self.view_max = str_readvec(f)
# Break up the reading a bit to limit the stack size.
(
flags,
bone_count,
bone_off,
bone_controller_count,
bone_controller_off,
hitbox_count,
hitbox_off,
anim_count,
anim_off,
sequence_count,
sequence_off,
) = struct_read('11I', f)
self.flags = Flags(flags)
(
activitylistversion,
eventsindexed,
texture_count,
texture_offset,
cdmat_count,
cdmat_offset,
skinref_count, # Number of skin "groups"
skin_count, # Number of model skins.
skinref_ind, # Location of skins reference table.
# The number of $body in the model (mstudiobodyparts_t).
bodypart_count,
bodypart_offset,
attachment_count,
attachment_offset,
) = struct_read('13i', f)
(
localnode_count,
localnode_index,
localnode_name_index,
# mstudioflexdesc_t
flexdesc_count,
flexdesc_index,
# mstudioflexcontroller_t
flexcontroller_count,
flexcontroller_index,
# mstudioflexrule_t
flexrules_count,
flexrules_index,
# IK probably refers to inverse kinematics
# mstudioikchain_t
ikchain_count,
ikchain_index,
# Information about any "mouth" on the model for speech animation
# More than one sounds pretty creepy.
# mstudiomouth_t
mouths_count,
mouths_index,
# mstudioposeparamdesc_t
localposeparam_count,
localposeparam_index,
) = struct_read('15I', f)
# VDC:
# For anyone trying to follow along, as of this writing,
# the next "surfaceprop_index" value is at position 0x0134 (308)
# from the start of the file.
assert f.tell() == 308, 'Offset wrong? {} != 308 {}'.format(
f.tell(), f)
(
# Surface property value (single null-terminated string)
surfaceprop_index,
# Unusual: In this one index comes first, then count.
# Key-value data is a series of strings. If you can't find
# what you're interested in, check the associated PHY file as well.
keyvalue_index,
keyvalue_count,
# More inverse-kinematics
# mstudioiklock_t
iklock_count,
iklock_index,
) = struct_read('5I', f)
(
self.mass, # Mass of object (float)
self.contents, # ??
# Other models can be referenced for re-used sequences and
# animations
# (See also: The $includemodel QC option.)
# mstudiomodelgroup_t
includemodel_count,
includemodel_index,
# In-engine, this is a pointer to the combined version of this +
# included models. In the file it's useless.
virtualModel,
# mstudioanimblock_t
animblocks_name_index,
animblocks_count,
animblocks_index,
animblockModel, # Placeholder for mutable-void*
# Points to a series of bytes?
bonetablename_index,
vertex_base, # Placeholder for void*
offset_base, # Placeholder for void*
) = struct_read('f 11I', f)
(
# Used with $constantdirectionallight from the QC
# Model should have flag #13 set if enabled
directionaldotproduct, # byte
# Preferred rather than clamped
rootLod, # byte
# 0 means any allowed, N means Lod 0 -> (N-1)
self.numAllowedRootLods, # byte
#unknown byte;
#unknown int;
# mstudioflexcontrollerui_t
flexcontrollerui_count,
flexcontrollerui_index,
) = struct_read('3b 5x 2I', f)
# Build CDMaterials data
f.seek(cdmat_offset)
self.cdmaterials = read_offset_array(f, cdmat_count)
for ind, cdmat in enumerate(self.cdmaterials):
cdmat = cdmat.replace('\\', '/').lstrip('/')
if cdmat and cdmat[-1:] != '/':
cdmat += '/'
self.cdmaterials[ind] = cdmat
# Build texture data
f.seek(texture_offset)
textures = [None] * texture_count # type: List[Tuple[str, int, int]]
tex_temp = [
None
] * texture_count # type: List[Tuple[int, Tuple[int, int, int]]]
for tex_ind in range(texture_count):
tex_temp[tex_ind] = (
f.tell(),
# Texture data:
# int: offset to the string, from start of struct.
# int: flags - appears to solely indicate 'teeth' materials...
# int: used, whatever that means.
# 4 unused bytes.
# 2 4-byte pointers in studiomdl to the material class, for
# server and client - shouldn't be in the file...
# 40 bytes of unused space (for expansion...)
struct_read('iii 4x 8x 40x', f))
for tex_ind, (offset, data) in enumerate(tex_temp):
name_offset, flags, used = data
textures[tex_ind] = (
read_nullstr(f, offset + name_offset),
flags,
used,
)
# Now parse through the family table, to match skins to textures.
f.seek(skinref_ind)
ref_data = f.read(2 * skinref_count * skin_count)
self.skins = [None] * skin_count # type: List[List[str]]
skin_group = Struct('<{}H'.format(skinref_count))
offset = 0
for ind in range(skin_count):
self.skins[ind] = [
textures[i][0].replace('\\', '/').lstrip('/')
for i in skin_group.unpack_from(ref_data, offset)
]
offset += skin_group.size
# If models have folders, add those folders onto cdmaterials.
for tex, flags, used in textures:
tex = tex.replace('\\', '/')
if '/' in tex:
folder = tex.rsplit('/', 1)[0]
if folder not in self.cdmaterials:
self.cdmaterials.append(folder)
# All models fallback to checking the texture at a root folder.
if '' not in self.cdmaterials:
self.cdmaterials.append('')
f.seek(surfaceprop_index)
self.surfaceprop = read_nullstr(f)
if keyvalue_count:
self.keyvalues = read_nullstr(f, keyvalue_index)
else:
self.keyvalues = ''
f.seek(includemodel_index)
self.included_models = [
None
] * includemodel_count # type: List[IncludedMDL]
for i in range(includemodel_count):
pos = f.tell()
# This is two offsets from the start of the structures.
lbl_pos, filename_pos = struct_read('II', f)
self.included_models[i] = IncludedMDL(
read_nullstr(f, pos + lbl_pos) if lbl_pos else '',
read_nullstr(f, pos + filename_pos) if filename_pos else '',
)
# Then return to after that struct - 4 bytes * 2.
f.seek(pos + 4 * 2)
f.seek(sequence_off)
self.sequences = self._read_sequences(f, sequence_count)
f.seek(bodypart_offset)
self._cull_skins_table(f, bodypart_count)
