def test_parse(py_c_token):
"""Test parsing strings."""
result = Property.parse(
# iter() ensures sequence methods aren't used anywhere.
iter(parse_test.splitlines()),
# Check active and inactive flags are correctly treated.
flags={
'test_enabled': True,
'test_disabled': False,
}
)
assert_tree(parse_result, result)
# Test the whole string can be passed too.
result = Property.parse(
parse_test,
flags={
'test_enabled': True,
'test_disabled': False,
},
)
assert_tree(parse_result, result)
# Check export roundtrips.
assert_tree(parse_result, Property.parse(parse_result.export()))
def pack_breakable_chunk(self, chunkname: str) -> None:
"""Pack the generic gib model for the given chunk name."""
if self._break_chunks is None:
# Need to load the file.
self.pack_file('scripts/propdata.txt')
try:
propdata = self.fsys['scripts/propdata.txt']
except FileNotFoundError:
LOGGER.warning('No scripts/propdata.txt for breakable chunks!')
return
with propdata.open_str() as f:
props = Property.parse(f,
'scripts/propdata.txt',
allow_escapes=False)
self._break_chunks = {}
for chunk_prop in props.find_children('BreakableModels'):
self._break_chunks[chunk_prop.name] = [
prop.real_name for prop in chunk_prop
]
try:
mdl_list = self._break_chunks[chunkname.casefold()]
except KeyError:
LOGGER.warning('Unknown gib chunks type "{}"!', chunkname)
return
for mdl in mdl_list:
self.pack_file(mdl, FileType.MODEL)
def load_soundscript(
self,
file: File,
*,
always_include: bool=False,
) -> Iterable[str]:
"""Read in a soundscript and record which files use it.
If always_include is True, it will be included in the manifests even
if it isn't used.
The sounds registered by this soundscript are returned.
"""
try:
with file.sys, file.open_str() as f:
props = Property.parse(f, file.path, allow_escapes=False)
except FileNotFoundError:
# It doesn't exist, complain and pretend it's empty.
LOGGER.warning('Soundscript "{}" does not exist!', file.path)
return ()
except KeyValError:
LOGGER.warning('Soundscript "{}" could not be parsed:', exc_info=True)
return ()
return self._parse_soundscript(props, file.path, always_include)
def read_prop(self, path: str, encoding='utf8') -> Property:
"""Read a Property file from the filesystem.
This handles opening and closing files.
"""
with self, self.open_str(path, encoding) as file:
return Property.parse(
file,
self.path + ':' + path,
)
def t(text):
"""Test a string to ensure it fails parsing."""
try:
result = Property.parse(text)
except KeyValError:
pass
else:
pytest.fail("Successfully parsed bad text ({!r}) to {!r}".format(
text,
result,
))
def load_soundscript(
self,
file: File,
*,
always_include: bool = False,
) -> Iterable[str]:
"""Read in a soundscript and record which files use it.
If always_include is True, it will be included in the manifests even
if it isn't used.
The sounds registered by this soundscript are returned.
"""
with file.sys, file.open_str() as f:
props = Property.parse(f, file.path)
return self._parse_soundscript(props, file.path, always_include)
def load_soundscript(
self,
file: File,
*,
always_include: bool=False,
) -> Iterable[str]:
"""Read in a soundscript and record which files use it.
If always_include is True, it will be included in the manifests even
if it isn't used.
The sounds registered by this soundscript are returned.
"""
with file.sys, file.open_str() as f:
props = Property.parse(f, file.path)
return self._parse_soundscript(props, file.path, always_include)
def parse_manifest(fsys: FileSystem, file: File=None) -> Dict[str, 'SurfaceProp']:
"""Load surfaceproperties from a manifest.
"scripts/surfaceproperties_manifest.txt" will be used if a file is
not specified.
"""
with fsys:
if not file:
file = fsys['scripts/surfaceproperties_manifest.txt']
with file.open_str() as f:
manifest = Property.parse(f, file.path)
surf = {}
for prop in manifest.find_all('surfaceproperties_manifest', 'file'):
surf = SurfaceProp.parse_file(fsys.read_prop(prop.value), surf)
return surf
def parse_manifest(fsys: FileSystem, file: File=None) -> Dict[str, 'SurfaceProp']:
"""Load surfaceproperties from a manifest.
"scripts/surfaceproperties_manifest.txt" will be used if a file is
not specified.
"""
with fsys:
if not file:
file = fsys['scripts/surfaceproperties_manifest.txt']
with file.open_str() as f:
manifest = Property.parse(f, file.path)
surf = {}
for prop in manifest.find_all('surfaceproperties_manifest', 'file'):
surf = SurfaceProp.parse_file(fsys.read_prop(prop.value), surf)
return surf
def load_soundscript_manifest(self, cache_file: str = None) -> None:
"""Read the soundscript manifest, and read all mentioned scripts.
If cache_file is provided, it should be a path to a file used to
cache the file reading for later use.
"""
try:
man = self.fsys.read_prop('scripts/game_sounds_manifest.txt')
except FileNotFoundError:
return
cache_data = {} # type: Dict[str, Tuple[int, Property]]
if cache_file is not None:
try:
f = open(cache_file)
except FileNotFoundError:
pass
else:
with f:
old_cache = Property.parse(f, cache_file)
for cache_prop in old_cache:
cache_data[cache_prop.name] = (
cache_prop.int('cache_key'),
cache_prop.find_key('files'))
# Regenerate from scratch each time - that way we remove old files
# from the list.
new_cache_data = Property(None, [])
else:
new_cache_data = None
with self.fsys:
for prop in man.find_children('game_sounds_manifest'):
if not prop.name.endswith('_file'):
continue
try:
cache_key, cache_files = cache_data[prop.value.casefold()]
except KeyError:
cache_key = -1
cache_files = None
file = self.fsys[prop.value]
cur_key = file.cache_key()
if cache_key != cur_key or cache_key == -1:
sounds = self.load_soundscript(file, always_include=True)
else:
# Read from cache.
sounds = []
for cache_prop in cache_files:
sounds.append(cache_prop.real_name)
self.soundscripts[cache_prop.real_name] = (
prop.value, [snd.value for snd in cache_prop])
# The soundscripts in the manifests are always included,
# since many would be part of the core code (physics, weapons,
# ui, etc). Just keep those loaded, no harm since vanilla does.
self.soundscript_files[file.path] = SoundScriptMode.INCLUDE
if new_cache_data is not None:
new_cache_data.append(
Property(prop.value, [
Property('cache_key', str(cur_key)),
Property('Files', [
Property(snd, [
Property('snd', raw)
for raw in self.soundscripts[snd][1]
]) for snd in sounds
])
]))
if cache_file is not None:
# Write back out our new cache with updated data.
with open(cache_file, 'w') as f:
for line in new_cache_data.export():
f.write(line)
def pack_file(
self,
filename: str,
data_type: FileType = FileType.GENERIC,
data: bytes = None,
) -> None:
"""Queue the given file to be packed.
If data is set, this file will use the given data instead of any
on-disk data. The data_type parameter allows specifying the kind of
file, which ensures it can be treated appropriately.
"""
filename = os.fspath(filename)
if '\t' in filename:
raise ValueError(
'No tabs are allowed in filenames ({!r})'.format(filename))
if data_type is FileType.GAME_SOUND:
self.pack_soundscript(filename)
return # This packs the soundscript and wav for us.
# If soundscript data is provided, load it and force-include it.
elif data_type is FileType.SOUNDSCRIPT and data:
self._parse_soundscript(
Property.parse(data.decode('utf8'), filename),
filename,
always_include=True,
)
if data_type is FileType.MATERIAL or (data_type is FileType.GENERIC
and filename.endswith('.vmt')):
data_type = FileType.MATERIAL
if not filename.startswith('materials/'):
filename = 'materials/' + filename
if not filename.endswith(('.vmt', '.spr')):
filename = filename + '.vmt'
elif data_type is FileType.TEXTURE or (data_type is FileType.GENERIC
and filename.endswith('.vtf')):
data_type = FileType.TEXTURE
if not filename.startswith('materials/'):
filename = 'materials/' + filename
if not filename.endswith('.vtf'):
filename = filename + '.vtf'
path = unify_path(filename)
try:
file = self._files[path]
except KeyError:
pass # We need to make it.
else:
# It's already here, is that OK?
# Allow overriding data on disk with ours..
if file.data is None:
if data is not None:
file.data = data
# else: no data on either, that's fine.
elif data == file.data:
pass # Overrode with the same data, that's fine
elif data:
raise ValueError(
'"{}": two different data streams!'.format(filename))
# else: we had an override, but asked to just pack now. That's fine.
if file.type is data_type:
# Same, no problems - just packing on top.
return
if file.type is FileType.GENERIC:
file.type = data_type # This is fine, we now know it has behaviour...
elif data_type is FileType.GENERIC:
pass # If we know it has behaviour already, that trumps generic.
elif data_type is FileType.WHITELIST:
file.type = data_type # Blindly believe this.
else:
raise ValueError('"{}": {} can\'t become a {}!'.format(
filename,
file.type.name,
data_type.name,
))
return # Don't re-add this.
start, ext = os.path.splitext(path)
# Try to promote generic to other types if known.
if data_type is FileType.GENERIC:
try:
data_type = EXT_TYPE[ext]
except KeyError:
pass
elif data_type is FileType.SOUNDSCRIPT:
if ext != '.txt':
raise ValueError(
'"{}" cannot be a soundscript!'.format(filename))
self._files[path] = PackFile(
data_type,
filename,
data,
)
def load_soundscript_manifest(self, cache_file: str=None) -> None:
"""Read the soundscript manifest, and read all mentioned scripts.
If cache_file is provided, it should be a path to a file used to
cache the file reading for later use.
"""
try:
man = self.fsys.read_prop('scripts/game_sounds_manifest.txt')
except FileNotFoundError:
return
cache_data = {} # type: Dict[str, Tuple[int, Property]]
if cache_file is not None:
# If the file doesn't exist or is corrupt, that's
# fine. We'll just parse the soundscripts the slow
# way.
try:
with open(cache_file) as f:
old_cache = Property.parse(f, cache_file)
if man['version'] != SOUND_CACHE_VERSION:
raise LookupError
except (FileNotFoundError, KeyValError, LookupError):
pass
else:
for cache_prop in old_cache.find_children('Sounds'):
cache_data[cache_prop.name] = (
cache_prop.int('cache_key'),
cache_prop.find_key('files')
)
# Regenerate from scratch each time - that way we remove old files
# from the list.
new_cache_sounds = Property('Sounds', [])
new_cache_data = Property(None, [
Property('version', SOUND_CACHE_VERSION),
new_cache_sounds,
])
else:
new_cache_data = new_cache_sounds = None
with self.fsys:
for prop in man.find_children('game_sounds_manifest'):
if not prop.name.endswith('_file'):
continue
try:
cache_key, cache_files = cache_data[prop.value.casefold()]
except KeyError:
cache_key = -1
cache_files = None
try:
file = self.fsys[prop.value]
except FileNotFoundError:
LOGGER.warning('Soundscript "{}" does not exist!', prop.value)
# Don't write anything into the cache, so we check this
# every time.
continue
cur_key = file.cache_key()
if cache_key != cur_key or cache_key == -1:
sounds = self.load_soundscript(file, always_include=True)
else:
# Read from cache.
sounds = []
for cache_prop in cache_files:
sounds.append(cache_prop.real_name)
self.soundscripts[cache_prop.real_name] = (prop.value, [
snd.value
for snd in cache_prop
])
# The soundscripts in the manifests are always included,
# since many would be part of the core code (physics, weapons,
# ui, etc). Just keep those loaded, no harm since vanilla does.
self.soundscript_files[file.path] = SoundScriptMode.INCLUDE
if new_cache_sounds is not None:
new_cache_sounds.append(Property(prop.value, [
Property('cache_key', str(cur_key)),
Property('Files', [
Property(snd, [
Property('snd', raw)
for raw in self.soundscripts[snd][1]
])
for snd in sounds
])
]))
if cache_file is not None:
# Write back out our new cache with updated data.
with srctools.AtomicWriter(cache_file) as f:
for line in new_cache_data.export():
f.write(line)
def pack_file(
self,
filename: str,
data_type: FileType=FileType.GENERIC,
data: bytes=None,
skinset: Set[int]=None,
optional: bool = False,
) -> None:
"""Queue the given file to be packed.
If data is set, this file will use the given data instead of any
on-disk data. The data_type parameter allows specifying the kind of
file, which ensures it can be treated appropriately.
If the file is a model, skinset allows restricting which skins are used.
If None (default), all skins may be used. Otherwise it is a set of
skins. If all uses of a model restrict the skin, only those skins need
to be packed.
If optional is set, this will be marked as optional so no errors occur
if it isn't in the filesystem.
"""
filename = os.fspath(filename)
# Assume an empty filename is an optional value.
if not filename:
if data is not None:
raise ValueError('Data provided with no filename!')
return
if '\t' in filename:
raise ValueError(
'No tabs are allowed in filenames ({!r})'.format(filename)
)
if data_type is FileType.GAME_SOUND:
self.pack_soundscript(filename)
return # This packs the soundscript and wav for us.
if data_type is FileType.PARTICLE:
# self.pack_particle(filename) # TODO: Particle parsing
return # This packs the PCF and material if required.
if data_type is FileType.CHOREO:
# self.pack_choreo(filename) # TODO: Choreo scene parsing
return
# If soundscript data is provided, load it and force-include it.
elif data_type is FileType.SOUNDSCRIPT and data:
self._parse_soundscript(
Property.parse(data.decode('utf8'), filename),
filename,
always_include=True,
)
filename = unify_path(filename)
if data_type is FileType.MATERIAL or (
data_type is FileType.GENERIC and filename.endswith('.vmt')
):
data_type = FileType.MATERIAL
if not filename.startswith('materials/'):
filename = 'materials/' + filename
if filename.endswith('.spr'):
# This is really wrong, spr materials don't exist anymore.
# Silently swap the extension.
filename = filename[:-3] + 'vmt'
elif not filename.endswith('.vmt'):
filename = filename + '.vmt'
elif data_type is FileType.TEXTURE or (
data_type is FileType.GENERIC and filename.endswith('.vtf')
):
data_type = FileType.TEXTURE
if not filename.startswith('materials/'):
filename = 'materials/' + filename
if not filename.endswith('.vtf'):
filename = filename + '.vtf'
elif data_type is FileType.VSCRIPT_SQUIRREL or (
data_type is FileType.GENERIC and filename.endswith('.nut')
):
data_type = FileType.VSCRIPT_SQUIRREL
if not filename.endswith('.nut'):
filename = filename + '.nut'
if data_type is FileType.MODEL or filename.endswith('.mdl'):
data_type = FileType.MODEL
if not filename.startswith('models/'):
filename = 'models/' + filename
if not filename.endswith('.mdl'):
filename = filename + '.mdl'
if skinset is None:
# It's dynamic, this overrides any previous specific skins.
self.skinsets[filename] = None
else:
try:
existing_skins = self.skinsets[filename]
except KeyError:
self.skinsets[filename] = skinset.copy()
else:
# Merge the two.
if existing_skins is not None:
self.skinsets[filename] = existing_skins | skinset
try:
file = self._files[filename]
except KeyError:
pass # We need to make it.
else:
# It's already here, is that OK?
# Allow overriding data on disk with ours..
if file.data is None:
if data is not None:
file.data = data
# else: no data on either, that's fine.
elif data == file.data:
pass # Overrode with the same data, that's fine
elif data:
raise ValueError('"{}": two different data streams!'.format(filename))
# else: we had an override, but asked to just pack now. That's fine.
# Override optional packing with required packing.
if not optional:
file.optional = False
if file.type is data_type:
# Same, no problems - just packing on top.
return
if file.type is FileType.GENERIC:
file.type = data_type # This is fine, we now know it has behaviour...
elif data_type is FileType.GENERIC:
pass # If we know it has behaviour already, that trumps generic.
else:
raise ValueError('"{}": {} can\'t become a {}!'.format(
filename,
file.type.name,
data_type.name,
))
return # Don't re-add this.
start, ext = os.path.splitext(filename)
# Try to promote generic to other types if known.
if data_type is FileType.GENERIC:
try:
data_type = EXT_TYPE[ext]
except KeyError:
pass
elif data_type is FileType.SOUNDSCRIPT:
if ext != '.txt':
raise ValueError('"{}" cannot be a soundscript!'.format(filename))
self._files[filename] = PackFile(data_type, filename, data, optional)
def load_soundscript_manifest(self, cache_file: str=None) -> None:
"""Read the soundscript manifest, and read all mentioned scripts.
If cache_file is provided, it should be a path to a file used to
cache the file reading for later use.
"""
try:
man = self.fsys.read_prop('scripts/game_sounds_manifest.txt')
except FileNotFoundError:
return
cache_data = {} # type: Dict[str, Tuple[int, Property]]
if cache_file is not None:
try:
f = open(cache_file)
except FileNotFoundError:
pass
else:
with f:
old_cache = Property.parse(f, cache_file)
for cache_prop in old_cache:
cache_data[cache_prop.name] = (
cache_prop.int('cache_key'),
cache_prop.find_key('files')
)
# Regenerate from scratch each time - that way we remove old files
# from the list.
new_cache_data = Property(None, [])
else:
new_cache_data = None
with self.fsys:
for prop in man.find_children('game_sounds_manifest'):
if not prop.name.endswith('_file'):
continue
try:
cache_key, cache_files = cache_data[prop.value.casefold()]
except KeyError:
cache_key = -1
cache_files = None
file = self.fsys[prop.value]
cur_key = file.cache_key()
if cache_key != cur_key or cache_key == -1:
sounds = self.load_soundscript(file, always_include=True)
else:
# Read from cache.
sounds = []
for cache_prop in cache_files:
sounds.append(cache_prop.real_name)
self.soundscripts[cache_prop.real_name] = (prop.value, [
snd.value
for snd in cache_prop
])
# The soundscripts in the manifests are always included,
# since many would be part of the core code (physics, weapons,
# ui, etc). Just keep those loaded, no harm since vanilla does.
self.soundscript_files[file.path] = SoundScriptMode.INCLUDE
if new_cache_data is not None:
new_cache_data.append(Property(prop.value, [
Property('cache_key', str(cur_key)),
Property('Files', [
Property(snd, [
Property('snd', raw)
for raw in self.soundscripts[snd][1]
])
for snd in sounds
])
]))
if cache_file is not None:
# Write back out our new cache with updated data.
with open(cache_file, 'w') as f:
for line in new_cache_data.export():
f.write(line)
def pack_file(
self,
filename: str,
data_type: FileType=FileType.GENERIC,
data: bytes=None,
) -> None:
"""Queue the given file to be packed.
If data is set, this file will use the given data instead of any
on-disk data. The data_type parameter allows specifying the kind of
file, which ensures it can be treated appropriately.
"""
filename = os.fspath(filename)
if '\t' in filename:
raise ValueError(
'No tabs are allowed in filenames ({!r})'.format(filename)
)
if data_type is FileType.GAME_SOUND:
self.pack_soundscript(filename)
return # This packs the soundscript and wav for us.
# If soundscript data is provided, load it and force-include it.
elif data_type is FileType.SOUNDSCRIPT and data:
self._parse_soundscript(
Property.parse(data.decode('utf8'), filename),
filename,
always_include=True,
)
if data_type is FileType.MATERIAL or (
data_type is FileType.GENERIC and filename.endswith('.vmt')
):
data_type = FileType.MATERIAL
if not filename.startswith('materials/'):
filename = 'materials/' + filename
if not filename.endswith(('.vmt', '.spr')):
filename = filename + '.vmt'
elif data_type is FileType.TEXTURE or (
data_type is FileType.GENERIC and filename.endswith('.vtf')
):
data_type = FileType.TEXTURE
if not filename.startswith('materials/'):
filename = 'materials/' + filename
if not filename.endswith('.vtf'):
filename = filename + '.vtf'
path = unify_path(filename)
try:
file = self._files[path]
except KeyError:
pass # We need to make it.
else:
# It's already here, is that OK?
# Allow overriding data on disk with ours..
if file.data is None:
if data is not None:
file.data = data
# else: no data on either, that's fine.
elif data == file.data:
pass # Overrode with the same data, that's fine
elif data:
raise ValueError('"{}": two different data streams!'.format(filename))
# else: we had an override, but asked to just pack now. That's fine.
if file.type is data_type:
# Same, no problems - just packing on top.
return
if file.type is FileType.GENERIC:
file.type = data_type # This is fine, we now know it has behaviour...
elif data_type is FileType.GENERIC:
pass # If we know it has behaviour already, that trumps generic.
elif data_type is FileType.WHITELIST:
file.type = data_type # Blindly believe this.
else:
raise ValueError('"{}": {} can\'t become a {}!'.format(
filename,
file.type.name,
data_type.name,
))
return # Don't re-add this.
start, ext = os.path.splitext(path)
# Try to promote generic to other types if known.
if data_type is FileType.GENERIC:
try:
data_type = EXT_TYPE[ext]
except KeyError:
pass
elif data_type is FileType.SOUNDSCRIPT:
if ext != '.txt':
raise ValueError('"{}" cannot be a soundscript!'.format(filename))
self._files[path] = PackFile(
data_type,
filename,
data,
)