def get_filesystem(self) -> FileSystemChain:
"""Build a chained filesystem from the search paths."""
vpks = []
raw_folders = []
for path in self.search_paths:
if path.is_dir():
raw_folders.append(path)
for ind in itertools.count(1):
vpk = (path / 'pak{:02}_dir.vpk'.format(ind))
if vpk.is_file():
vpks.append(vpk)
else:
break
continue
if not path.suffix:
path = path.with_suffix('.vpk')
if not path.name.endswith('_dir.vpk'):
path = path.with_name(path.name[:-4] + '_dir.vpk')
if path.is_file() and path.suffix == '.vpk':
vpks.append(path)
fsys = FileSystemChain()
for path in vpks:
fsys.add_sys(VPKFileSystem(path))
for path in raw_folders:
fsys.add_sys(RawFileSystem(path))
return fsys
def get_filesystem(self) -> FileSystemChain:
"""Build a chained filesystem from the search paths."""
vpks = []
raw_folders = []
for path in self.search_paths:
if path.is_dir():
raw_folders.append(path)
if (path / 'pak01_dir.vpk').is_file():
vpks.append(path / 'pak01_dir.vpk')
continue
if not path.suffix:
path = path.with_suffix('.vpk')
if not path.name.endswith('_dir.vpk'):
path = path.with_name(path.name[:-4] + '_dir.vpk')
if path.is_file() and path.suffix == '.vpk':
vpks.append(path)
fsys = FileSystemChain()
for path in vpks:
fsys.add_sys(VPKFileSystem(path))
for path in raw_folders:
fsys.add_sys(RawFileSystem(path))
return fsys
def parse(path: Path) -> Tuple[
Config,
Game,
FileSystemChain,
Set[FileSystem],
Set[Plugin],
]:
"""From some directory, locate and parse the config file.
This then constructs and customises each object according to config
options.
The first srctools.vdf file found in a parent directory is parsed.
If none can be found, it tries to find the first subfolder of 'common/' and
writes a default copy there. FileNotFoundError is raised if none can be
found.
This returns:
* The config.
* Parsed gameinfo.
* The chain of filesystems.
* A packing blacklist.
* A list of plugins.
"""
conf = Config(OPTIONS)
# If the path is a folder, add a dummy folder so parents yields it.
# That way we check for a config in this folder.
if not path.suffix:
path /= 'unused'
for folder in path.parents:
conf_path = folder / CONF_NAME
if conf_path.exists():
LOGGER.info('Config path: "{}"', conf_path.absolute())
with open(conf_path) as f:
props = Property.parse(f, conf_path)
conf.path = conf_path
conf.load(props)
break
else:
LOGGER.warning('Cannot find a valid config file!')
# Apply all the defaults.
conf.load(Property(None, []))
# Try to write out a default file in the game folder.
for folder in path.parents:
if folder.parent.stem == 'common':
break
else:
# Give up, write to working directory.
folder = Path()
conf.path = folder / CONF_NAME
LOGGER.warning('Writing default to "{}"', conf.path)
with AtomicWriter(str(conf.path)) as f:
conf.save(f)
game = Game((folder / conf.get(str, 'gameinfo')).resolve())
fsys_chain = game.get_filesystem()
blacklist = set() # type: Set[FileSystem]
if not conf.get(bool, 'pack_vpk'):
for fsys, prefix in fsys_chain.systems:
if isinstance(fsys, VPKFileSystem):
blacklist.add(fsys)
game_root = game.root
for prop in conf.get(Property, 'searchpaths'): # type: Property
if prop.has_children():
raise ValueError('Config "searchpaths" value cannot have children.')
assert isinstance(prop.value, str)
if prop.value.endswith('.vpk'):
fsys = VPKFileSystem(str((game_root / prop.value).resolve()))
else:
fsys = RawFileSystem(str((game_root / prop.value).resolve()))
if prop.name in ('prefix', 'priority'):
fsys_chain.add_sys(fsys, priority=True)
elif prop.name == 'nopack':
blacklist.add(fsys)
elif prop.name in ('path', 'pack'):
fsys_chain.add_sys(fsys)
else:
raise ValueError(
'Unknown searchpath '
'key "{}"!'.format(prop.real_name)
)
plugins = set() # type: Set[Plugin]
# find all the plugins and make plugin objects out of them
for prop in conf.get(Property, 'plugins'): # type: Property
if prop.has_children():
raise ValueError('Config "plugins" value cannot have children.')
assert isinstance(prop.value, str)
path = (game_root / Path(prop.value)).resolve()
if prop.name in ("path", "recursive"):
if not path.is_dir():
raise ValueError("'{}' is not a directory".format(path))
# want to recursive glob if key is recursive
pattern = "*.py" if prop.name == "path" else "**/*.py"
#find all .py files, make Plugins
for p in path.glob(pattern):
plugins.add(Plugin(path / p))
elif prop.name == "single":
plugins.add(Plugin(path))
else:
raise ValueError("Unknown plugins key {}".format(prop.real_name))
return conf, game, fsys_chain, blacklist, plugins
def main(argv: List[str]) -> None:
LOGGER.info('BEE2 VRAD hook started!')
args = " ".join(argv)
fast_args = argv[1:]
full_args = argv[1:]
if not fast_args:
# No arguments!
LOGGER.info(
'No arguments!\n'
"The BEE2 VRAD takes all the regular VRAD's "
'arguments, with some extra arguments:\n'
'-force_peti: Force enabling map conversion. \n'
"-force_hammer: Don't convert the map at all.\n"
"If not specified, the map name must be \"preview.bsp\" to be "
"treated as PeTI.")
sys.exit()
# The path is the last argument to vrad
# P2 adds wrong slashes sometimes, so fix that.
fast_args[-1] = path = os.path.normpath(argv[-1]) # type: str
LOGGER.info("Map path is " + path)
load_config()
for a in fast_args[:]:
if a.casefold() in (
"-both",
"-final",
"-staticproplighting",
"-staticproppolys",
"-textureshadows",
):
# remove final parameters from the modified arguments
fast_args.remove(a)
elif a in ('-force_peti', '-force_hammer', '-no_pack'):
# we need to strip these out, otherwise VRAD will get confused
fast_args.remove(a)
full_args.remove(a)
fast_args = ['-bounce', '2', '-noextra'] + fast_args
# Fast args: -bounce 2 -noextra -game $gamedir $path\$file
# Final args: -both -final -staticproplighting -StaticPropPolys
# -textureshadows -game $gamedir $path\$file
if not path.endswith(".bsp"):
path += ".bsp"
if not os.path.exists(path):
raise ValueError('"{}" does not exist!'.format(path))
if not os.path.isfile(path):
raise ValueError('"{}" is not a file!'.format(path))
# If VBSP thinks it's hammer, trust it.
if CONF.bool('is_hammer', False):
is_peti = edit_args = False
else:
is_peti = True
# Detect preview via knowing the bsp name. If we are in preview,
# check the config file to see what was specified there.
if os.path.basename(path) == "preview.bsp":
edit_args = not CONF.bool('force_full', False)
else:
# publishing - always force full lighting.
edit_args = False
if '-force_peti' in args or '-force_hammer' in args:
# we have override commands!
if '-force_peti' in args:
LOGGER.warning('OVERRIDE: Applying cheap lighting!')
is_peti = edit_args = True
else:
LOGGER.warning('OVERRIDE: Preserving args!')
is_peti = edit_args = False
LOGGER.info('Final status: is_peti={}, edit_args={}', is_peti, edit_args)
# Grab the currently mounted filesystems in P2.
game = find_gameinfo(argv)
root_folder = game.path.parent
fsys = game.get_filesystem()
fsys_tag = fsys_mel = None
if is_peti and 'mel_vpk' in CONF:
fsys_mel = VPKFileSystem(CONF['mel_vpk'])
fsys.add_sys(fsys_mel)
if is_peti and 'tag_dir' in CONF:
fsys_tag = RawFileSystem(CONF['tag_dir'])
fsys.add_sys(fsys_tag)
LOGGER.info('Reading BSP')
bsp_file = BSP(path)
bsp_ents = bsp_file.read_ent_data()
zip_data = BytesIO()
zip_data.write(bsp_file.get_lump(BSP_LUMPS.PAKFILE))
zipfile = ZipFile(zip_data, mode='a')
# Mount the existing packfile, so the cubemap files are recognised.
fsys.systems.append((ZipFileSystem('', zipfile), ''))
fsys.open_ref()
LOGGER.info('Done!')
LOGGER.info('Reading our FGD files...')
fgd = load_fgd()
packlist = PackList(fsys)
packlist.load_soundscript_manifest(
str(root_folder / 'bin/bee2/sndscript_cache.vdf'))
# We nee to add all soundscripts in scripts/bee2_snd/
# This way we can pack those, if required.
for soundscript in fsys.walk_folder('scripts/bee2_snd/'):
if soundscript.path.endswith('.txt'):
packlist.load_soundscript(soundscript, always_include=False)
if is_peti:
LOGGER.info('Adding special packed files:')
music_data = CONF.find_key('MusicScript', [])
if music_data:
packlist.pack_file('scripts/BEE2_generated_music.txt',
PackType.SOUNDSCRIPT,
data=generate_music_script(
music_data, packlist))
for filename, arcname in inject_files():
LOGGER.info('Injecting "{}" into packfile.', arcname)
with open(filename, 'rb') as f:
packlist.pack_file(arcname, data=f.read())
LOGGER.info('Run transformations...')
run_transformations(bsp_ents, fsys, packlist)
LOGGER.info('Scanning map for files to pack:')
packlist.pack_from_bsp(bsp_file)
packlist.pack_fgd(bsp_ents, fgd)
packlist.eval_dependencies()
LOGGER.info('Done!')
if is_peti:
packlist.write_manifest()
else:
# Write with the map name, so it loads directly.
packlist.write_manifest(os.path.basename(path)[:-4])
# We need to disallow Valve folders.
pack_whitelist = set() # type: Set[FileSystem]
pack_blacklist = set() # type: Set[FileSystem]
if is_peti:
pack_blacklist |= {
RawFileSystem(root_folder / 'portal2_dlc2'),
RawFileSystem(root_folder / 'portal2_dlc1'),
RawFileSystem(root_folder / 'portal2'),
RawFileSystem(root_folder / 'platform'),
RawFileSystem(root_folder / 'update'),
}
if fsys_mel is not None:
pack_whitelist.add(fsys_mel)
if fsys_tag is not None:
pack_whitelist.add(fsys_tag)
if '-no_pack' not in args:
# Cubemap files packed into the map already.
existing = set(zipfile.infolist())
LOGGER.info('Writing to BSP...')
packlist.pack_into_zip(
zipfile,
ignore_vpk=True,
whitelist=pack_whitelist,
blacklist=pack_blacklist,
)
LOGGER.info(
'Packed files:\n{}', '\n'.join([
zipinfo.filename for zipinfo in zipfile.infolist()
if zipinfo.filename not in existing
]))
dump_files(zipfile)
zipfile.close() # Finalise the zip modification
# Copy the zipfile into the BSP file, and adjust the headers.
bsp_file.lumps[BSP_LUMPS.PAKFILE].data = zip_data.getvalue()
# Copy new entity data.
bsp_file.lumps[BSP_LUMPS.ENTITIES].data = BSP.write_ent_data(bsp_ents)
bsp_file.save()
LOGGER.info(' - BSP written!')
if is_peti:
mod_screenshots()
if edit_args:
LOGGER.info("Forcing Cheap Lighting!")
run_vrad(fast_args)
else:
if is_peti:
LOGGER.info(
"Publishing - Full lighting enabled! (or forced to do so)")
else:
LOGGER.info("Hammer map detected! Not forcing cheap lighting..")
run_vrad(full_args)
LOGGER.info("BEE2 VRAD hook finished!")
def _parse_template(loc: UnparsedTemplate) -> Template:
"""Parse a template VMF."""
filesys: FileSystem
if os.path.isdir(loc.pak_path):
filesys = RawFileSystem(loc.pak_path)
else:
ext = os.path.splitext(loc.pak_path)[1].casefold()
if ext in ('.bee_pack', '.zip'):
filesys = ZipFileSystem(loc.pak_path)
elif ext == '.vpk':
filesys = VPKFileSystem(loc.pak_path)
else:
raise ValueError(f'Unknown filesystem type for "{loc.pak_path}"!')
with filesys[loc.path].open_str() as f:
props = Property.parse(f, f'{loc.pak_path}:{loc.path}')
vmf = srctools.VMF.parse(props, preserve_ids=True)
del props, filesys, f # Discard all this data.
# visgroup -> list of brushes/overlays
detail_ents: dict[str, list[Solid]] = defaultdict(list)
world_ents: dict[str, list[Solid]] = defaultdict(list)
overlay_ents: dict[str, list[Entity]] = defaultdict(list)
color_pickers: list[ColorPicker] = []
tile_setters: list[TileSetter] = []
voxel_setters: list[VoxelSetter] = []
conf_ents = vmf.by_class['bee2_template_conf']
if len(conf_ents) > 1:
raise ValueError(
f'Multiple configuration entities in template "{loc.id}"!')
elif not conf_ents:
raise ValueError(f'No configration entity for template "{loc.id}"!')
else:
[conf] = conf_ents
if conf['template_id'].upper() != loc.id:
raise ValueError(
f'Mismatch in template IDs for {conf["template_id"]} and {loc.id}')
def yield_world_detail() -> Iterator[tuple[list[Solid], bool, set[str]]]:
"""Yield all world/detail solids in the map.
This also indicates if it's a func_detail, and the visgroup IDs.
(Those are stored in the ent for detail, and the solid for world.)
"""
for brush in vmf.brushes:
yield [brush], False, brush.visgroup_ids
for detail in vmf.by_class['func_detail']:
yield detail.solids, True, detail.visgroup_ids
force = conf['temp_type']
force_is_detail: Optional[bool]
if force.casefold() == 'detail':
force_is_detail = True
elif force.casefold() == 'world':
force_is_detail = False
else:
force_is_detail = None
visgroup_names = {vis.id: vis.name.casefold() for vis in vmf.vis_tree}
conf_auto_visgroup = 1 if srctools.conv_bool(
conf['detail_auto_visgroup']) else 0
if not srctools.conv_bool(conf['discard_brushes']):
for brushes, is_detail, vis_ids in yield_world_detail():
visgroups = list(map(visgroup_names.__getitem__, vis_ids))
if len(visgroups) > 1:
raise ValueError('Template "{}" has brush with two '
'visgroups! ({})'.format(
loc.id, ', '.join(visgroups)))
# No visgroup = ''
visgroup = visgroups[0] if visgroups else ''
# Auto-visgroup puts func_detail ents in unique visgroups.
if is_detail and not visgroup and conf_auto_visgroup:
visgroup = '__auto_group_{}__'.format(conf_auto_visgroup)
# Reuse as the unique index, >0 are True too..
conf_auto_visgroup += 1
# Check this after auto-visgroup, so world/detail can be used to
# opt into the grouping, then overridden to be the same.
if force_is_detail is not None:
is_detail = force_is_detail
if is_detail:
detail_ents[visgroup].extend(brushes)
else:
world_ents[visgroup].extend(brushes)
for ent in vmf.by_class['info_overlay']:
visgroups = list(map(visgroup_names.__getitem__, ent.visgroup_ids))
if len(visgroups) > 1:
raise ValueError('Template "{}" has overlay with two '
'visgroups! ({})'.format(loc.id,
', '.join(visgroups)))
# No visgroup = ''
overlay_ents[visgroups[0] if visgroups else ''].append(ent)
for ent in vmf.by_class['bee2_template_colorpicker']:
# Parse the colorpicker data.
try:
priority = Decimal(ent['priority'])
except ArithmeticError:
LOGGER.warning(
'Bad priority for colorpicker in "{}" template!',
loc.id,
)
priority = Decimal(0)
try:
remove_after = AfterPickMode(ent['remove_brush', '0'])
except ValueError:
LOGGER.warning(
'Bad remove-brush mode for colorpicker in "{}" template!',
loc.id,
)
remove_after = AfterPickMode.NONE
color_pickers.append(
ColorPicker(
priority=priority,
name=ent['targetname'],
visgroups=set(map(visgroup_names.__getitem__,
ent.visgroup_ids)),
offset=Vec.from_str(ent['origin']),
normal=Vec(x=1) @ Angle.from_str(ent['angles']),
sides=ent['faces'].split(' '),
grid_snap=srctools.conv_bool(ent['grid_snap']),
after=remove_after,
use_pattern=srctools.conv_bool(ent['use_pattern']),
force_tex_white=ent['tex_white'],
force_tex_black=ent['tex_black'],
))
for ent in vmf.by_class['bee2_template_voxelsetter']:
tile_type = TILE_SETTER_SKINS[srctools.conv_int(ent['skin'])]
voxel_setters.append(
VoxelSetter(
offset=Vec.from_str(ent['origin']),
normal=Vec(z=1) @ Angle.from_str(ent['angles']),
visgroups=set(map(visgroup_names.__getitem__,
ent.visgroup_ids)),
tile_type=tile_type,
force=srctools.conv_bool(ent['force']),
))
for ent in vmf.by_class['bee2_template_tilesetter']:
tile_type = TILE_SETTER_SKINS[srctools.conv_int(ent['skin'])]
color = ent['color']
if color == 'tile':
try:
color = tile_type.color
except ValueError:
# Non-tile types.
color = None
elif color == 'invert':
color = 'INVERT'
elif color == 'match':
color = None
elif color != 'copy':
raise ValueError('Invalid TileSetter color '
'"{}" for "{}"'.format(color, loc.id))
tile_setters.append(
TileSetter(
offset=Vec.from_str(ent['origin']),
normal=Vec(z=1) @ Angle.from_str(ent['angles']),
visgroups=set(map(visgroup_names.__getitem__,
ent.visgroup_ids)),
color=color,
tile_type=tile_type,
picker_name=ent['color_picker'],
force=srctools.conv_bool(ent['force']),
))
coll: list[CollisionDef] = []
for ent in vmf.by_class['bee2_collision_bbox']:
visgroups = set(map(visgroup_names.__getitem__, ent.visgroup_ids))
for bbox in collisions.BBox.from_ent(ent):
coll.append(CollisionDef(bbox, visgroups))
return Template(
loc.id,
set(visgroup_names.values()),
world_ents,
detail_ents,
overlay_ents,
conf['skip_faces'].split(),
conf['realign_faces'].split(),
conf['overlay_faces'].split(),
conf['vertical_faces'].split(),
color_pickers,
tile_setters,
voxel_setters,
coll,
)
def parse(
path: Path
) -> Tuple[Config, Game, FileSystemChain, Set[FileSystem], PluginFinder, ]:
"""From some directory, locate and parse the config file.
This then constructs and customises each object according to config
options.
The first srctools.vdf file found in a parent directory is parsed.
If none can be found, it tries to find the first subfolder of 'common/' and
writes a default copy there. FileNotFoundError is raised if none can be
found.
This returns:
* The config.
* Parsed gameinfo.
* The chain of filesystems.
* A packing blacklist.
* The plugin loader.
"""
conf = Config(OPTIONS)
# If the path is a folder, add a dummy folder so parents yields it.
# That way we check for a config in this folder.
if not path.suffix:
path /= 'unused'
for folder in path.parents:
conf_path = folder / CONF_NAME
if conf_path.exists():
LOGGER.info('Config path: "{}"', conf_path.absolute())
with open(conf_path) as f:
props = Property.parse(f, conf_path)
conf.path = conf_path
conf.load(props)
break
else:
LOGGER.warning('Cannot find a valid config file!')
# Apply all the defaults.
conf.load(Property(None, []))
# Try to write out a default file in the game folder.
for folder in path.parents:
if folder.parent.stem == 'common':
break
else:
# Give up, write to working directory.
folder = Path()
conf.path = folder / CONF_NAME
LOGGER.warning('Writing default to "{}"', conf.path)
with AtomicWriter(str(conf.path)) as f:
conf.save(f)
game = Game((folder / conf.get(str, 'gameinfo')).resolve())
fsys_chain = game.get_filesystem()
blacklist = set() # type: Set[FileSystem]
if not conf.get(bool, 'pack_vpk'):
for fsys, prefix in fsys_chain.systems:
if isinstance(fsys, VPKFileSystem):
blacklist.add(fsys)
game_root = game.root
for prop in conf.get(Property, 'searchpaths'): # type: Property
if prop.has_children():
raise ValueError(
'Config "searchpaths" value cannot have children.')
assert isinstance(prop.value, str)
if prop.value.endswith('.vpk'):
fsys = VPKFileSystem(str((game_root / prop.value).resolve()))
else:
fsys = RawFileSystem(str((game_root / prop.value).resolve()))
if prop.name in ('prefix', 'priority'):
fsys_chain.add_sys(fsys, priority=True)
elif prop.name == 'nopack':
blacklist.add(fsys)
elif prop.name in ('path', 'pack'):
fsys_chain.add_sys(fsys)
else:
raise ValueError('Unknown searchpath '
'key "{}"!'.format(prop.real_name))
sources: Dict[Path, PluginSource] = {}
builtin_transforms = (Path(sys.argv[0]).parent / 'transforms').resolve()
# find all the plugins and make plugin objects out of them
for prop in conf.get(Property, 'plugins'):
if prop.has_children():
raise ValueError('Config "plugins" value cannot have children.')
assert isinstance(prop.value, str)
path = (game_root / Path(prop.value)).resolve()
if prop.name in ('path', "recursive", 'folder'):
if not path.is_dir():
raise ValueError("'{}' is not a directory".format(path))
is_recursive = prop.name == "recursive"
try:
source = sources[path]
except KeyError:
sources[path] = PluginSource(path, is_recursive)
else:
if is_recursive and not source.recursive:
# Upgrade to recursive.
source.recursive = True
elif prop.name in ('single', 'file'):
parent = path.parent
try:
source = sources[parent]
except KeyError:
source = sources[parent] = PluginSource(parent, False)
source.autoload_files.add(path)
elif prop.name == "_builtin_":
# For development purposes, redirect builtin folder.
builtin_transforms = path
else:
raise ValueError("Unknown plugins key {}".format(prop.real_name))
for source in sources.values():
LOGGER.debug('Plugin path: "{}", recursive={}, files={}',
source.folder, source.recursive,
sorted(source.autoload_files))
LOGGER.debug('Builtin plugin path is {}', builtin_transforms)
if builtin_transforms not in sources:
sources[builtin_transforms] = PluginSource(builtin_transforms, True)
plugin_finder = PluginFinder('srctools.bsp_transforms.plugin',
sources.values())
sys.meta_path.append(plugin_finder)
return conf, game, fsys_chain, blacklist, plugin_finder
def find_packages(pak_dir: str) -> None:
"""Search a folder for packages, recursing if necessary."""
found_pak = False
for name in os.listdir(pak_dir): # Both files and dirs
name = os.path.join(pak_dir, name)
folded = name.casefold()
if folded.endswith('.vpk') and not folded.endswith('_dir.vpk'):
# _000.vpk files, useless without the directory
continue
if os.path.isdir(name):
filesys = RawFileSystem(name)
else:
ext = os.path.splitext(folded)[1]
if ext in ('.bee_pack', '.zip'):
filesys = ZipFileSystem(name)
elif ext == '.vpk':
filesys = VPKFileSystem(name)
else:
LOGGER.info('Extra file: {}', name)
continue
LOGGER.debug('Reading package "' + name + '"')
# Gain a persistent hold on the filesystem's handle.
# That means we don't need to reopen the zip files constantly.
filesys.open_ref()
# Valid packages must have an info.txt file!
try:
info = filesys.read_prop('info.txt')
except FileNotFoundError:
# Close the ref we've gotten, since it's not in the dict
# it won't be done by load_packages().
filesys.close_ref()
if os.path.isdir(name):
# This isn't a package, so check the subfolders too...
LOGGER.debug('Checking subdir "{}" for packages...', name)
find_packages(name)
else:
LOGGER.warning('ERROR: package "{}" has no info.txt!', name)
# Don't continue to parse this "package"
continue
try:
pak_id = info['ID']
except IndexError:
# Close the ref we've gotten, since it's not in the dict
# it won't be done by load_packages().
filesys.close_ref()
raise
if pak_id in packages:
raise ValueError(
f'Duplicate package with id "{pak_id}"!\n'
'If you just updated the mod, delete any old files in packages/.'
) from None
PACKAGE_SYS[pak_id] = filesys
packages[pak_id] = Package(
pak_id,
filesys,
info,
name,
)
found_pak = True
if not found_pak:
LOGGER.info('No packages in folder {}!', pak_dir)