def load_fgd() -> FGD:
"""Extract the local copy of FGD data.
This allows the analysis to not depend on local files.
"""
from lzma import LZMAFile
with LZMAFile(open_binary(srctools, 'fgd.lzma')) as f:
return FGD.unserialise(f)
def load_fgd() -> FGD:
"""Extract the local copy of FGD data.
This allows the analysis to not depend on local files.
"""
from lzma import LZMAFile
with LZMAFile(open_binary(srctools, 'fgd.lzma')) as f:
return FGD.unserialise(f)
def load_fgd() -> FGD:
"""Extract the local copy of FGD data.
This allows the analysis to not depend on local files.
"""
import warnings
warnings.warn(
'Use FGD.engine_dbase() instead, '
'this has been moved there.',
DeprecationWarning,
stacklevel=2,
)
return FGD.engine_dbase()
def action_import(
dbase: Path,
engine_tag: str,
fgd_paths: List[Path],
) -> None:
"""Import an FGD file, adding differences to the unified files."""
new_fgd = FGD()
print('Using tag "{}"'.format(engine_tag))
expanded = expand_tags(frozenset({engine_tag}))
print('Reading {} FGDs:'.format(len(fgd_paths)))
for path in fgd_paths:
print(path)
with RawFileSystem(str(path.parent)) as fsys:
new_fgd.parse_file(fsys, fsys[path.name], eval_bases=False)
print('\nImporting {} entiti{}...'.format(
len(new_fgd),
"y" if len(new_fgd) == 1 else "ies",
))
for new_ent in new_fgd:
path = dbase / ent_path(new_ent)
path.parent.mkdir(parents=True, exist_ok=True)
if path.exists():
old_fgd = FGD()
with RawFileSystem(str(path.parent)) as fsys:
old_fgd.parse_file(fsys, fsys[path.name], eval_bases=False)
try:
ent = old_fgd[new_ent.classname]
except KeyError:
raise ValueError("Classname not present in FGD!")
# Now merge the two.
if new_ent.desc not in ent.desc:
# Temporary, append it.
ent.desc += '|||' + new_ent.desc
# Merge helpers. We just combine overall...
for new_base in new_ent.bases:
if new_base not in ent.bases:
ent.bases.append(new_base)
for helper in new_ent.helpers:
# Sorta ew, quadratic search. But helper sizes shouldn't
# get too big.
if helper not in ent.helpers:
ent.helpers.append(helper)
for cat in ('keyvalues', 'inputs', 'outputs'):
cur_map = getattr(
ent, cat
) # type: Dict[str, Dict[FrozenSet[str], Union[KeyValues, IODef]]]
new_map = getattr(new_ent, cat)
new_names = set()
for name, tag_map in new_map.items():
new_names.add(name)
try:
orig_tag_map = cur_map[name]
except KeyError:
# Not present in the old file.
cur_map[name] = {
add_tag(tag, engine_tag): value
for tag, value in tag_map.items()
}
continue
# Otherwise merge, if unequal add the new ones.
# TODO: Handle tags in "new" files.
for tag, new_value in tag_map.items():
for old_tag, old_value in orig_tag_map.items():
if old_value == new_value:
if tag:
# Already present, modify this tag.
del orig_tag_map[old_tag]
orig_tag_map[add_tag(
old_tag, engine_tag)] = new_value
# else: Blank tag, keep blank.
break
else:
# Otherwise, we need to add this.
orig_tag_map[add_tag(tag, engine_tag)] = new_value
# Make sure removed items don't apply to the new tag.
for name, tag_map in cur_map.items():
if name not in new_names:
cur_map[name] = {
add_tag(tag, '!' + engine_tag): value
for tag, value in tag_map.items()
}
else:
# No existing one, just set appliesto.
ent = new_ent
applies_to = get_appliesto(ent)
if not match_tags(expanded, applies_to):
applies_to.append(engine_tag)
ent.helpers[:] = [
helper for helper in ent.helpers
if not isinstance(helper, HelperExtAppliesTo)
]
with open(path, 'w') as f:
ent.export(f)
print('.', end='', flush=True)
print()
def load_database(dbase: Path,
extra_loc: Path = None,
fgd_vis: bool = False) -> Tuple[FGD, EntityDef]:
"""Load the entire database from disk. This returns the FGD, plus the CBaseEntity definition."""
print(f'Loading database {dbase}:')
fgd = FGD()
fgd.map_size_min = -16384
fgd.map_size_max = 16384
# Classname -> filename
ent_source: Dict[str, str] = {}
with RawFileSystem(str(dbase)) as fsys:
for file in dbase.rglob("*.fgd"):
# Use a temp FGD class, to allow us to verify no overwrites.
file_fgd = FGD()
rel_loc = str(file.relative_to(dbase))
file_fgd.parse_file(
fsys,
fsys[rel_loc],
eval_bases=False,
encoding='utf8',
)
for clsname, ent in file_fgd.entities.items():
if clsname in fgd.entities:
raise ValueError(
f'Duplicate "{clsname}" class '
f'in {rel_loc} and {ent_source[clsname]}!')
fgd.entities[clsname] = ent
ent_source[clsname] = rel_loc
if fgd_vis:
for parent, visgroup in file_fgd.auto_visgroups.items():
try:
existing_group = fgd.auto_visgroups[parent]
except KeyError:
fgd.auto_visgroups[parent] = visgroup
else: # Need to merge
existing_group.ents.update(visgroup.ents)
fgd.mat_exclusions.update(file_fgd.mat_exclusions)
for tags, mat_list in file_fgd.tagged_mat_exclusions.items():
fgd.tagged_mat_exclusions[tags] |= mat_list
print('.', end='', flush=True)
load_visgroup_conf(fgd, dbase)
if extra_loc is not None:
print('\nLoading extra file:')
if extra_loc.is_file():
# One file.
with RawFileSystem(str(extra_loc.parent)) as fsys:
fgd.parse_file(
fsys,
fsys[extra_loc.name],
eval_bases=False,
)
else:
print('\nLoading extra files:')
with RawFileSystem(str(extra_loc)) as fsys:
for file in extra_loc.rglob("*.fgd"):
fgd.parse_file(
fsys,
fsys[str(file.relative_to(extra_loc))],
eval_bases=False,
)
print('.', end='', flush=True)
print()
fgd.apply_bases()
print('\nDone!')
print('Entities without visgroups:')
vis_ents = {
name.casefold()
for group in fgd.auto_visgroups.values() for name in group.ents
}
vis_count = ent_count = 0
for ent in fgd:
# Base ents, worldspawn, or engine-only ents don't need visgroups.
if ent.type is EntityTypes.BASE or ent.classname == 'worldspawn':
continue
applies_to = get_appliesto(ent)
if '+ENGINE' in applies_to or 'ENGINE' in applies_to:
continue
ent_count += 1
if ent.classname.casefold() not in vis_ents:
print(ent.classname, end=', ')
else:
vis_count += 1
print(
f'\nVisgroup count: {vis_count}/{ent_count} ({vis_count*100/ent_count:.2f}%) done!'
)
try:
base_entity_def = fgd.entities.pop(BASE_ENTITY.casefold())
base_entity_def.type = EntityTypes.BASE
except KeyError:
base_entity_def = EntityDef(EntityTypes.BASE)
return fgd, base_entity_def
def main(argv: List[str]) -> None:
parser = argparse.ArgumentParser(
description="Modifies the BSP file, allowing additional entities "
"and bugfixes.", )
parser.add_argument("--nopack",
dest="allow_pack",
action="store_false",
help="Prevent packing of files found in the map.")
parser.add_argument(
"--propcombine",
action="store_true",
help="Allow merging static props together.",
)
parser.add_argument(
"--showgroups",
action="store_true",
help="Show propcombined props, by setting their tint to random groups",
)
parser.add_argument(
"--dumpgroups",
action="store_true",
help="Write all props without propcombine groups to a new VMF.",
)
parser.add_argument(
"map",
help="The path to the BSP file.",
)
args = parser.parse_args(argv)
# The path is the last argument to the compiler.
# Hammer adds wrong slashes sometimes, so fix that.
# Also if it's the VMF file, make it the BSP.
path = Path(args.map).with_suffix('.bsp')
# Open and start writing to the map's log file.
handler = FileHandler(path.with_suffix('.log'))
handler.setFormatter(
Formatter(
# One letter for level name
'[{levelname}] {module}.{funcName}(): {message}',
style='{',
))
LOGGER.addHandler(handler)
LOGGER.info('Srctools postcompiler hook started at {}!',
datetime.datetime.now().isoformat())
LOGGER.info("Map path is {}", path)
conf, game_info, fsys, pack_blacklist, plugin = config.parse(path)
fsys.open_ref()
packlist = PackList(fsys)
LOGGER.info('Gameinfo: {}', game_info.path)
LOGGER.info(
'Search paths: \n{}',
'\n'.join([sys.path for sys, prefix in fsys.systems]),
)
fgd = FGD.engine_dbase()
LOGGER.info('Loading soundscripts...')
packlist.load_soundscript_manifest(
conf.path.with_name('srctools_sndscript_data.vdf'))
LOGGER.info('Done! ({} sounds)', len(packlist.soundscripts))
LOGGER.info('Reading BSP...')
bsp_file = BSP(path)
LOGGER.info('Reading entities...')
vmf = bsp_file.read_ent_data()
LOGGER.info('Done!')
# Mount the existing packfile, so the cubemap files are recognised.
LOGGER.info('Mounting BSP packfile...')
zipfile = ZipFile(BytesIO(bsp_file.get_lump(BSP_LUMPS.PAKFILE)))
fsys.add_sys(ZipFileSystem('<BSP pakfile>', zipfile))
studiomdl_path = conf.get(str, 'studiomdl')
if studiomdl_path:
studiomdl_loc = (game_info.root / studiomdl_path).resolve()
if not studiomdl_loc.exists():
LOGGER.warning('No studiomdl found at "{}"!', studiomdl_loc)
studiomdl_loc = None
else:
LOGGER.warning('No studiomdl path provided.')
studiomdl_loc = None
LOGGER.info('Loading plugins...')
plugin.load_all()
use_comma_sep = conf.get(bool, 'use_comma_sep')
if use_comma_sep is None:
# Guess the format, by picking whatever the first output uses.
for ent in vmf.entities:
for out in ent.outputs:
use_comma_sep = out.comma_sep
break
if use_comma_sep is None:
LOGGER.warning(
'No outputs in map, could not determine BSP I/O format!')
LOGGER.warning('Set "use_comma_sep" in srctools.vdf.')
use_comma_sep = False
LOGGER.info('Running transforms...')
run_transformations(vmf, fsys, packlist, bsp_file, game_info,
studiomdl_loc)
if studiomdl_loc is not None and args.propcombine:
decomp_cache_loc = conf.get(str, 'propcombine_cache')
if decomp_cache_loc is not None:
decomp_cache_loc = (game_info.root / decomp_cache_loc).resolve()
decomp_cache_loc.mkdir(parents=True, exist_ok=True)
if conf.get(bool, 'propcombine_crowbar'):
# argv[0] is the location of our script/exe, which lets us locate
# Crowbar from there.
crowbar_loc = Path(sys.argv[0], '../Crowbar.exe').resolve()
else:
crowbar_loc = None
LOGGER.info('Combining props...')
propcombine.combine(
bsp_file,
vmf,
packlist,
game_info,
studiomdl_loc=studiomdl_loc,
qc_folders=[
game_info.root / folder for folder in conf.get(
Property, 'propcombine_qc_folder').as_array(conv=Path)
],
decomp_cache_loc=decomp_cache_loc,
crowbar_loc=crowbar_loc,
auto_range=conf.get(int, 'propcombine_auto_range'),
min_cluster=conf.get(int, 'propcombine_min_cluster'),
debug_tint=args.showgroups,
debug_dump=args.dumpgroups,
)
LOGGER.info('Done!')
else: # Strip these if they're present.
for ent in vmf.by_class['comp_propcombine_set']:
ent.remove()
bsp_file.lumps[BSP_LUMPS.ENTITIES].data = bsp_file.write_ent_data(
vmf, use_comma_sep)
if conf.get(bool, 'auto_pack') and args.allow_pack:
LOGGER.info('Analysing packable resources...')
packlist.pack_fgd(vmf, fgd)
packlist.pack_from_bsp(bsp_file)
packlist.eval_dependencies()
if conf.get(bool, 'soundscript_manifest'):
packlist.write_manifest()
packlist.pack_into_zip(bsp_file,
blacklist=pack_blacklist,
ignore_vpk=False)
with bsp_file.packfile() as pak_zip:
# List out all the files, but group together files with the same extension.
ext_for_name: Dict[str, List[str]] = defaultdict(list)
for file in pak_zip.infolist():
filename = Path(file.filename)
if '.' in filename.name:
stem, ext = filename.name.split('.', 1)
file_path = str(filename.parent / stem)
else:
file_path = file.filename
ext = ''
ext_for_name[file_path].append(ext)
LOGGER.info('Packed files: \n{}'.format('\n'.join([
(f'{name}.{exts[0]}'
if len(exts) == 1 else f'{name}.({"/".join(exts)})')
for name, exts in sorted(ext_for_name.items())
])))
LOGGER.info('Writing BSP...')
bsp_file.save()
LOGGER.info("srctools VRAD hook finished!")
def load_database(dbase: Path, extra_loc: Path = None) -> FGD:
"""Load the entire database from disk."""
print('Loading database:')
fgd = FGD()
fgd.map_size_min = -16384
fgd.map_size_max = 16384
with RawFileSystem(str(dbase)) as fsys:
for file in dbase.rglob("*.fgd"):
fgd.parse_file(
fsys,
fsys[str(file.relative_to(dbase))],
eval_bases=False,
)
print('.', end='', flush=True)
if extra_loc is not None:
print('\nLoading extra file:')
if extra_loc.is_file():
# One file.
with RawFileSystem(str(extra_loc.parent)) as fsys:
fgd.parse_file(
fsys,
fsys[extra_loc.name],
eval_bases=False,
)
else:
print('\nLoading extra files:')
with RawFileSystem(str(extra_loc)) as fsys:
for file in extra_loc.rglob("*.fgd"):
fgd.parse_file(
fsys,
fsys[str(file.relative_to(extra_loc))],
eval_bases=False,
)
print('.', end='', flush=True)
print()
fgd.apply_bases()
print('\nDone!')
return fgd
def main(argv: List[str]) -> None:
parser = argparse.ArgumentParser(
description="Modifies the BSP file, allowing additional entities "
"and bugfixes.", )
parser.add_argument("--nopack",
dest="allow_pack",
action="store_false",
help="Prevent packing of files found in the map.")
parser.add_argument(
"--propcombine",
action="store_true",
help="Allow merging static props together.",
)
parser.add_argument(
"--showgroups",
action="store_true",
help="Show propcombined props, by setting their tint to 0 255 0",
)
parser.add_argument(
"map",
help="The path to the BSP file.",
)
args = parser.parse_args(argv)
# The path is the last argument to the compiler.
# Hammer adds wrong slashes sometimes, so fix that.
# Also if it's the VMF file, make it the BSP.
path = Path(args.map).with_suffix('.bsp')
# Open and start writing to the map's log file.
handler = FileHandler(path.with_suffix('.log'))
handler.setFormatter(
Formatter(
# One letter for level name
'[{levelname}] {module}.{funcName}(): {message}',
style='{',
))
LOGGER.addHandler(handler)
LOGGER.info('Srctools postcompiler hook started at {}!',
datetime.datetime.now().isoformat())
LOGGER.info("Map path is {}", path)
conf, game_info, fsys, pack_blacklist, plugins = config.parse(path)
fsys.open_ref()
packlist = PackList(fsys)
LOGGER.info('Gameinfo: {}', game_info.path)
LOGGER.info(
'Search paths: \n{}',
'\n'.join([sys.path for sys, prefix in fsys.systems]),
)
fgd = FGD.engine_dbase()
LOGGER.info('Loading soundscripts...')
packlist.load_soundscript_manifest(
conf.path.with_name('srctools_sndscript_data.vdf'))
LOGGER.info('Done! ({} sounds)', len(packlist.soundscripts))
LOGGER.info('Reading BSP...')
bsp_file = BSP(path)
LOGGER.info('Reading entities...')
vmf = bsp_file.read_ent_data()
LOGGER.info('Done!')
studiomdl_path = conf.get(str, 'studiomdl')
if studiomdl_path:
studiomdl_loc = (game_info.root / studiomdl_path).resolve()
if not studiomdl_loc.exists():
LOGGER.warning('No studiomdl found at "{}"!', studiomdl_loc)
studiomdl_loc = None
else:
LOGGER.warning('No studiomdl path provided.')
studiomdl_loc = None
for plugin in plugins:
plugin.load()
use_comma_sep = conf.get(bool, 'use_comma_sep')
if use_comma_sep is None:
# Guess the format, by picking whatever the first output uses.
for ent in vmf.entities:
for out in ent.outputs:
use_comma_sep = out.comma_sep
break
if use_comma_sep is None:
LOGGER.warning(
'No outputs in map, could not determine BSP I/O format!')
LOGGER.warning('Set "use_comma_sep" in srctools.vdf.')
use_comma_sep = False
run_transformations(vmf, fsys, packlist, bsp_file, game_info,
studiomdl_loc)
if studiomdl_loc is not None and args.propcombine:
LOGGER.info('Combining props...')
propcombine.combine(
bsp_file,
vmf,
packlist,
game_info,
studiomdl_loc,
[
game_info.root / folder for folder in conf.get(
Property, 'propcombine_qc_folder').as_array(conv=Path)
],
conf.get(int, 'propcombine_auto_range'),
conf.get(int, 'propcombine_min_cluster'),
debug_tint=args.showgroups,
)
LOGGER.info('Done!')
else: # Strip these if they're present.
for ent in vmf.by_class['comp_propcombine_set']:
ent.remove()
bsp_file.lumps[BSP_LUMPS.ENTITIES].data = bsp_file.write_ent_data(
vmf, use_comma_sep)
if conf.get(bool, 'auto_pack') and args.allow_pack:
LOGGER.info('Analysing packable resources...')
packlist.pack_fgd(vmf, fgd)
packlist.pack_from_bsp(bsp_file)
packlist.eval_dependencies()
packlist.pack_into_zip(bsp_file,
blacklist=pack_blacklist,
ignore_vpk=False)
with bsp_file.packfile() as pak_zip:
LOGGER.info('Packed files: \n{}'.format('\n'.join(pak_zip.namelist())))
LOGGER.info('Writing BSP...')
bsp_file.save()
LOGGER.info("srctools VRAD hook finished!")