def init_trans(self): """Try and load a copy of basemodui from Portal 2 to translate. Valve's items use special translation strings which would look ugly if we didn't convert them. """ # Already loaded if TRANS_DATA: return # We need to first figure out what language is used (if not English), # then load in the file. This is saved in the 'appmanifest', try: appman_file = open(self.abs_path('../../appmanifest_620.acf')) except FileNotFoundError: # Portal 2 isn't here... return with appman_file: appman = Property.parse(appman_file, 'appmanifest_620.acf') try: lang = appman.find_key('AppState').find_key( 'UserConfig')['language'] except NoKeyError: return basemod_loc = self.abs_path( '../Portal 2/portal2_dlc2/resource/basemodui_' + lang + '.txt') # Basemod files are encoded in UTF-16. try: basemod_file = open(basemod_loc, encoding='utf16') except FileNotFoundError: return with basemod_file: if lang == 'english': def filterer(file): """The English language has some unused language text. This needs to be skipped since it has invalid quotes.""" for line in file: if line.count('"') <= 4: yield line basemod_file = filterer(basemod_file) trans_prop = Property.parse(basemod_file, 'basemodui.txt') for item in trans_prop.find_key("lang", []).find_key("tokens", []): TRANS_DATA[item.real_name] = item.value
def init_trans(self): """Try and load a copy of basemodui from Portal 2 to translate. Valve's items use special translation strings which would look ugly if we didn't convert them. """ # Already loaded if TRANS_DATA: return # We need to first figure out what language is used (if not English), # then load in the file. This is saved in the 'appmanifest', try: appman_file = open(self.abs_path("../../appmanifest_620.acf")) except FileNotFoundError: # Portal 2 isn't here... return with appman_file: appman = Property.parse(appman_file, "appmanifest_620.acf") try: lang = appman.find_key("AppState").find_key("UserConfig")["language"] except NoKeyError: return basemod_loc = self.abs_path("../Portal 2/portal2_dlc2/resource/basemodui_" + lang + ".txt") # Basemod files are encoded in UTF-16. try: basemod_file = open(basemod_loc, encoding="utf16") except FileNotFoundError: return with basemod_file: if lang == "english": def filterer(file): """The English language has some unused language text. This needs to be skipped since it has invalid quotes.""" for line in file: if line.count('"') <= 4: yield line basemod_file = filterer(basemod_file) trans_prop = Property.parse(basemod_file, "basemodui.txt") for item in trans_prop.find_key("lang", []).find_key("tokens", []): TRANS_DATA[item.real_name] = item.value
def clean_text(file_path): # Try and parse as a property file. If it succeeds, # write that out - it removes excess whitespace between lines with open(file_path, 'r') as f: try: props = Property.parse(f) except KeyValError: pass else: for line in props.export(): yield line.lstrip() return with open(file_path, 'r') as f: for line in f: if line.isspace(): continue if line.lstrip().startswith('//'): continue # Remove // comments, but only if the comment doesn't have # a quote char after it - it could be part of the string, # so leave it just to be safe. if '//' in line and '"' not in line: yield line.split('//')[0] + '\n' else: yield line.lstrip()
def load_trans(self, lang): """Actually load the translation.""" # Already loaded if TRANS_DATA: return basemod_loc = self.abs_path( '../Portal 2/portal2_dlc2/resource/basemodui_' + lang + '.txt') # Basemod files are encoded in UTF-16. try: basemod_file = open(basemod_loc, encoding='utf16') except FileNotFoundError: return with basemod_file: if lang == 'english': def filterer(file): """The English language has some unused language text. This needs to be skipped since it has invalid quotes.""" for line in file: if line.count('"') <= 4: yield line basemod_file = filterer(basemod_file) trans_prop = Property.parse(basemod_file, 'basemodui.txt') for item in trans_prop.find_key("lang", []).find_key("tokens", []): TRANS_DATA[item.real_name] = item.value
def get_intersect_testcases() -> list: """Use a VMF to make it easier to generate the bounding boxes.""" with Path(__file__, '../bbox_samples.vmf').open() as f: vmf = VMF.parse(Property.parse(f)) def process(brush: Solid | None) -> tuple[tuple[int, ...], tuple[int, ...]] | None: """Extract the bounding box from the brush.""" if brush is None: return None bb_min, bb_max = brush.get_bbox() for vec in [bb_min, bb_max]: for ax in 'xyz': # If one thick, make zero thick so we can test planes. if abs(vec[ax]) == 63: vec[ax] = math.copysign(64, vec[ax]) return (tuple(map(int, bb_min)), tuple(map(int, bb_max))) for ent in vmf.entities: test = expected = None for solid in ent.solids: if solid.sides[0].mat.casefold() == 'tools/toolsskip': expected = solid if solid.sides[0].mat.casefold() == 'tools/toolstrigger': test = solid if test is None: raise ValueError(ent.id) yield (*process(test), process(expected))
def load_templates(): """Load in the template file, used for import_template().""" with open(TEMPLATE_LOCATION) as file: props = Property.parse(file, TEMPLATE_LOCATION) vmf = srctools.VMF.parse(props, preserve_ids=True) def make_subdict(): return defaultdict(list) # detail_ents[temp_id][visgroup] detail_ents = defaultdict(make_subdict) world_ents = defaultdict(make_subdict) overlay_ents = defaultdict(make_subdict) conf_ents = {} for ent in vmf.by_class['bee2_template_world']: world_ents[ent['template_id'].casefold()][ ent['visgroup'].casefold()].extend(ent.solids) for ent in vmf.by_class['bee2_template_detail']: detail_ents[ent['template_id'].casefold()][ ent['visgroup'].casefold()].extend(ent.solids) for ent in vmf.by_class['bee2_template_overlay']: overlay_ents[ent['template_id'].casefold()][ ent['visgroup'].casefold()].append(ent) for ent in vmf.by_class['bee2_template_conf']: conf_ents[ent['template_id'].casefold()] = ent for ent in vmf.by_class['bee2_template_scaling']: temp = ScalingTemplate.parse(ent) TEMPLATES[temp.id.casefold()] = temp for temp_id in set(detail_ents).union(world_ents, overlay_ents): try: conf = conf_ents[temp_id] except KeyError: overlay_faces = [] skip_faces = [] vertical_faces = [] realign_faces = [] else: vertical_faces = conf['vertical_faces'].split() realign_faces = conf['realign_faces'].split() overlay_faces = conf['overlay_faces'].split() skip_faces = conf['skip_faces'].split() TEMPLATES[temp_id.casefold()] = Template( temp_id, world_ents[temp_id], detail_ents[temp_id], overlay_ents[temp_id], skip_faces, realign_faces, overlay_faces, vertical_faces, )
def read_settings() -> None: """Read and apply the settings from disk.""" try: file = open(utils.conf_location('config/config.vdf'), encoding='utf8') except FileNotFoundError: return with file: props = Property.parse(file) apply_settings(props)
def load_config(): global CONF LOGGER.info('Loading Settings...') try: with open("bee2/vrad_config.cfg", encoding='utf8') as config: CONF = Property.parse(config, 'bee2/vrad_config.cfg').find_key( 'Config', []) except FileNotFoundError: pass LOGGER.info('Config Loaded!')
def load_config(): global CONF LOGGER.info('Loading Settings...') try: with open("bee2/vrad_config.cfg", encoding='utf8') as config: CONF = Property.parse(config, 'bee2/vrad_config.cfg').find_key( 'Config', [] ) except FileNotFoundError: pass LOGGER.info('Config Loaded!')
def loader() -> Property: """Load and parse the specified file when called.""" try: with file.open_str() as f: props = Property.parse(f) except (KeyValError, FileNotFoundError, UnicodeDecodeError): LOGGER.exception('Unable to read "{}"', path) raise if source: packages.set_cond_source(props, source) return props
def __init__(self, path: Union[str, Path]): """Parse a game from a folder.""" if isinstance(path, Path): self.path = path else: self.path = Path(path) with open(self.path / GINFO) as f: gameinfo = Property.parse(f).find_key('GameInfo') fsystems = gameinfo.find_key('Filesystem', []) self.game_name = gameinfo['Game'] self.app_id = fsystems['SteamAppId'] self.tools_id = fsystems['ToolsAppId', None] self.additional_content = fsystems['AdditionalContentId', None] self.fgd_loc = gameinfo['GameData', 'None'] self.search_paths = [] # type: List[Path] for path in fsystems.find_children('SearchPaths'): exp_path = self.parse_search_path(path) # Expand /* if at the end of paths. if exp_path.name == '*': try: self.search_paths.extend( map(exp_path.parent.joinpath, os.listdir(exp_path.parent))) except FileNotFoundError: pass # Handle folder_* too. elif exp_path.name.endswith('*'): exp_path = exp_path.with_name(exp_path.name[:-1]) self.search_paths.extend( filter(Path.is_dir, exp_path.glob(exp_path.name))) else: self.search_paths.append(exp_path) # Add DLC folders based on the first/bin folder. try: first_search = self.search_paths[0] except IndexError: pass else: folder = first_search.parent stem = first_search.name + '_dlc' for ind in itertools.count(1): path = folder / (stem + str(ind)) if path.exists(): self.search_paths.insert(0, path) else: break # Force including 'platform', for Hammer assets. self.search_paths.append(self.path.parent / 'platform')
def gen_sound_manifest(additional, excludes): """Generate a new game_sounds_manifest.txt file. This includes all the current scripts defined, plus any custom ones. Excludes is a list of scripts to remove from the listing - this allows overriding the sounds without VPK overrides. """ if not additional: return # Don't pack, there aren't any new sounds.. orig_manifest = os.path.join( '..', SOUND_MAN_FOLDER.get(CONF['game_id', ''], 'portal2'), 'scripts', 'game_sounds_manifest.txt', ) try: with open(orig_manifest) as f: props = Property.parse(f, orig_manifest).find_key( 'game_sounds_manifest', [], ) except FileNotFoundError: # Assume no sounds props = Property('game_sounds_manifest', []) scripts = [prop.value for prop in props.find_all('precache_file')] for script in additional: scripts.append(script) for script in excludes: try: scripts.remove(script) except ValueError: LOGGER.warning( '"{}" should be excluded, but it\'s' ' not in the manifest already!', script, ) # Build and unbuild it to strip other things out - Valve includes a bogus # 'new_sound_scripts_must_go_below_here' entry.. new_props = Property('game_sounds_manifest', [Property('precache_file', file) for file in scripts]) inject_loc = os.path.join('bee2', 'inject', 'soundscript_manifest.txt') with open(inject_loc, 'w') as f: for line in new_props.export(): f.write(line) LOGGER.info('Written new soundscripts_manifest..')
def from_file(path: utils.PackagePath, missing_ok: bool=False, source: str= '') -> LazyConf: """Lazily load the specified config.""" try: fsys = packages.PACKAGE_SYS[path.package] except KeyError: if not missing_ok: LOGGER.warning('Package does not exist: "{}"', path) return BLANK try: file = fsys[path.path] except FileNotFoundError: if not missing_ok: LOGGER.warning('File does not exist: "{}"', path) return BLANK def loader() -> Property: """Load and parse the specified file when called.""" try: with file.open_str() as f: props = Property.parse(f) except (KeyValError, FileNotFoundError, UnicodeDecodeError): LOGGER.exception('Unable to read "{}"', path) raise if source: packages.set_cond_source(props, source) return props if DEV_MODE.get(): # Parse immediately, to check syntax. try: with file.open_str() as f: Property.parse(f) except (KeyValError, FileNotFoundError, UnicodeDecodeError): LOGGER.exception('Unable to read "{}"', path) return loader
def from_file(cls, path, zip_file): """Initialise from a file. path is the file path for the map inside the zip, without extension. zip_file is either a ZipFile or FakeZip object. """ # Some P2Cs may have non-ASCII characters in descriptions, so we # need to read it as bytes and convert to utf-8 ourselves - zips # don't convert encodings automatically for us. try: with zip_open_bin(zip_file, path + '.p2c') as file: props = Property.parse( # Decode the P2C as UTF-8, and skip unknown characters. # We're only using it for display purposes, so that should # be sufficent. TextIOWrapper( file, encoding='utf-8', errors='replace', ), path, ) except KeyValError: # Silently fail if we can't parse the file. That way it's still # possible to backup. LOGGER.warning('Failed parsing puzzle file!', path, exc_info=True) props = Property('portal2_puzzle', []) title = None desc = _('Failed to parse this puzzle file. It can still be backed up.') else: props = props.find_key('portal2_puzzle', []) title = props['title', None] desc = props['description', _('No description found.')] if title is None: title = '<' + path.rsplit('/', 1)[-1] + '.p2c>' return cls( filename=os.path.basename(path), zip_file=zip_file, title=title, desc=desc, is_coop=srctools.conv_bool(props['coop', '0']), create_time=Date(props['timestamp_created', '']), mod_time=Date(props['timestamp_modified', '']), )
def parse(cls, path: str): with open(path) as f: props = Property.parse(f, path) name = props['Name', '??'] items = [] for item in props.find_children('Items'): items.append((item.real_name, int(item.value))) trans_name = props['TransName', ''] return Palette( name, items, trans_name=trans_name, prevent_overwrite=props.bool('readonly'), filename=os.path.basename(path), )
def read_settings() -> None: """Read and apply the settings from disk.""" path = utils.conf_location('config/config.vdf') try: file = path.open(encoding='utf8') except FileNotFoundError: return try: with file: props = Property.parse(file) except KeyValError: LOGGER.warning('Cannot parse config.vdf!', exc_info=True) # Try and move to a backup name, if not don't worry about it. try: path.replace(path.with_suffix('.err.vdf')) except IOError: pass apply_settings(props)
def _parse_phy(self, f: BinaryIO, filename: str) -> None: """Parse the physics data file, if present. """ [ size, header_id, solid_count, checksum, ] = ST_PHY_HEADER.unpack(f.read(ST_PHY_HEADER.size)) f.read(size - ST_PHY_HEADER.size) # If the header is larger ever. for solid in range(solid_count): [solid_size] = struct_read('i', f) f.read(solid_size) # Skip the header. self.phys_keyvalues = Property.parse( read_nullstr(f), filename + ":keyvalues", allow_escapes=False, single_line=True, )
def parse(cls, path: str) -> Palette: """Parse a palette from a file.""" needs_save = False with open(path, encoding='utf8') as f: props = Property.parse(f, path) name = props['Name', '??'] items = [] for item in props.find_children('Items'): items.append((item.real_name, int(item.value))) trans_name = props['TransName', ''] if trans_name: # Builtin, force a fixed uuid. This is mainly for LAST_EXPORT. uuid = uuid5(DEFAULT_NS, trans_name) else: try: uuid = UUID(hex=props['UUID']) except (ValueError, LookupError): uuid = uuid4() needs_save = True settings: Property | None try: settings = props.find_key('Settings') except NoKeyError: settings = None pal = Palette( name, items, trans_name=trans_name, group=props['group', ''], readonly=props.bool('readonly'), filename=os.path.basename(path), uuid=uuid, settings=settings, ) if needs_save: LOGGER.info('Resaving older palette file {}', pal.filename) pal.save() return pal
def gen_vpks(): with open('vpk/vpk_dest.cfg') as f: config = Property.parse(f, 'vpk/vpk_dest.cfg').find_key("VPKDest", []) if not os.path.isfile(VPK_BIN_LOC): print('VPK.exe not present, skipping VPK generation.') return for prop in config: src = os.path.join('vpk', prop.real_name) dest = os.path.abspath('packages/{}/{}.vpk'.format(prop.value, src)) subprocess.call([ VPK_BIN_LOC, src, ]) if os.path.isfile(dest): os.remove(dest) os.rename(src + '.vpk', dest) print('Processed "{}"'.format(dest))
def parse_legacy(posfile, propfile, path): """Parse the original BEE2.2 palette format.""" props = Property.parse(propfile, path + ':properties.txt') name = props['name', 'Unnamed'] pos = [] for dirty_line in posfile: line = srctools.clean_line(dirty_line) if line: # Lines follow the form # "ITEM_BUTTON_FLOOR", 2 # for subtype 3 of the button if line.startswith('"'): val = line.split('",') if len(val) == 2: pos.append(( val[0][1:], # Item ID int(val[1].strip()), # Item subtype )) else: LOGGER.warning('Malformed row "{}"!', line) return None return Palette(name, pos)
def gen_part_manifest(additional): """Generate a new particle system manifest file. This includes all the current ones defined, plus any custom ones. """ if not additional: return # Don't pack, there aren't any new particles.. orig_manifest = os.path.join( '..', GAME_FOLDER.get(CONF['game_id', ''], 'portal2'), 'particles', 'particles_manifest.txt', ) try: with open(orig_manifest) as f: props = Property.parse(f, orig_manifest).find_key( 'particles_manifest', [], ) except FileNotFoundError: # Assume no particles props = Property('particles_manifest', []) parts = [prop.value for prop in props.find_all('file')] for particle in additional: parts.append(particle) # Build and unbuild it to strip comments and similar lines. new_props = Property('particles_manifest', [Property('file', file) for file in parts]) inject_loc = os.path.join('bee2', 'inject', 'particles_manifest.txt') with open(inject_loc, 'w') as f: for line in new_props.export(): f.write(line) LOGGER.info('Written new particles_manifest..')
def __init__(self, path: Union[str, Path]): """Parse a game from a folder.""" if isinstance(path, Path): self.path = path else: self.path = Path(path) with open(self.path / GINFO) as f: gameinfo = Property.parse(f).find_key('GameInfo') fsystems = gameinfo.find_key('Filesystem', []) self.game_name = gameinfo['Game'] self.app_id = fsystems['SteamAppId'] self.tools_id = fsystems['ToolsAppId', None] self.additional_content = fsystems['AdditionalContentId', None] self.fgd_loc = gameinfo['GameData', 'None'] self.search_paths = [] # type: List[Path] for path in fsystems.find_children('SearchPaths'): self.search_paths.append( self.parse_search_path(self.path.parent, path)) # Add DLC folders based on the first/bin folder. try: first_search = self.search_paths[0] except IndexError: pass else: folder = first_search.parent stem = first_search.name + '_dlc' for ind in itertools.count(1): path = folder / (stem + str(ind)) if path.exists(): self.search_paths.insert(0, path) else: break # Force including 'platform', for Hammer assets. self.search_paths.append(self.path.parent / 'platform')
def parse(cls, path: str): with open(path, encoding='utf8') as f: props = Property.parse(f, path) name = props['Name', '??'] items = [] for item in props.find_children('Items'): items.append((item.real_name, int(item.value))) trans_name = props['TransName', ''] try: settings = props.find_key('Settings') except NoKeyError: settings = None return Palette( name, items, trans_name=trans_name, prevent_overwrite=props.bool('readonly'), filename=os.path.basename(path), settings=settings, )
def init_trans(self): """Try and load a copy of basemodui from Portal 2 to translate. Valve's items use special translation strings which would look ugly if we didn't convert them. """ # Already loaded if TRANS_DATA: return # Allow overriding. try: lang = os.environ['BEE2_P2_LANG'] except KeyError: pass else: self.load_trans(lang) return # We need to first figure out what language is used (if not English), # then load in the file. This is saved in the 'appmanifest', try: appman_file = open(self.abs_path('../../appmanifest_620.acf')) except FileNotFoundError: # Portal 2 isn't here... return with appman_file: appman = Property.parse(appman_file, 'appmanifest_620.acf') try: lang = appman.find_key('AppState').find_key( 'UserConfig')['language'] except NoKeyError: return self.load_trans(lang)
def __init__(self, path: Union[str, Path]): """Parse a game from a folder.""" if isinstance(path, Path): self.path = path else: self.path = Path(path) with open(self.path / GINFO) as f: gameinfo = Property.parse(f).find_key('GameInfo') fsystems = gameinfo.find_key('Filesystem', []) self.game_name = gameinfo['Game'] self.app_id = fsystems['SteamAppId'] self.tools_id = fsystems['ToolsAppId', None] self.additional_content = fsystems['AdditionalContentId', None] self.fgd_loc = gameinfo['GameData', 'None'] self.search_paths = [] # type: List[Path] for path in fsystems.find_children('SearchPaths'): self.search_paths.append(self.parse_search_path(self.path.parent, path)) # Add DLC folders based on the first/bin folder. try: first_search = self.search_paths[0] except IndexError: pass else: folder = first_search.parent stem = first_search.name + '_dlc' for ind in itertools.count(1): path = folder / (stem + str(ind)) if path.exists(): self.search_paths.insert(0, path) else: break # Force including 'platform', for Hammer assets. self.search_paths.append(self.path.parent / 'platform')
def libraryFolders() -> list: """ Retrieves the steam library folders by parsing the libraryfolders.vdf file :return: a list with all library paths """ paths = [steamDir() + '/steamapps/'] # create a list for library paths try: # open the file that contains the library paths with open(steamDir() + '/steamapps/libraryfolders.vdf', 'r') as file: library = Property.parse(file, 'libraryfolders.vdf').as_dict() # remove useless stuff library['libraryfolders'].pop('timenextstatsreport') library['libraryfolders'].pop('contentstatsid') except Exception as e: raise Exception(f'Error while reading steam library file: {e}') # check for other library paths, if the dict is empty, there's no one if len(library['libraryfolders']) != 0: for i in range(len(library['libraryfolders'])): paths.append(library['libraryfolders'][i] + '/steamapps/') # append the path # return the "compiled" list of libraries return paths
def parse(path: Path) -> Tuple[ Config, Game, FileSystemChain, Set[FileSystem], Set[Plugin], ]: """From some directory, locate and parse the config file. This then constructs and customises each object according to config options. The first srctools.vdf file found in a parent directory is parsed. If none can be found, it tries to find the first subfolder of 'common/' and writes a default copy there. FileNotFoundError is raised if none can be found. This returns: * The config. * Parsed gameinfo. * The chain of filesystems. * A packing blacklist. * A list of plugins. """ conf = Config(OPTIONS) # If the path is a folder, add a dummy folder so parents yields it. # That way we check for a config in this folder. if not path.suffix: path /= 'unused' for folder in path.parents: conf_path = folder / CONF_NAME if conf_path.exists(): LOGGER.info('Config path: "{}"', conf_path.absolute()) with open(conf_path) as f: props = Property.parse(f, conf_path) conf.path = conf_path conf.load(props) break else: LOGGER.warning('Cannot find a valid config file!') # Apply all the defaults. conf.load(Property(None, [])) # Try to write out a default file in the game folder. for folder in path.parents: if folder.parent.stem == 'common': break else: # Give up, write to working directory. folder = Path() conf.path = folder / CONF_NAME LOGGER.warning('Writing default to "{}"', conf.path) with AtomicWriter(str(conf.path)) as f: conf.save(f) game = Game((folder / conf.get(str, 'gameinfo')).resolve()) fsys_chain = game.get_filesystem() blacklist = set() # type: Set[FileSystem] if not conf.get(bool, 'pack_vpk'): for fsys, prefix in fsys_chain.systems: if isinstance(fsys, VPKFileSystem): blacklist.add(fsys) game_root = game.root for prop in conf.get(Property, 'searchpaths'): # type: Property if prop.has_children(): raise ValueError('Config "searchpaths" value cannot have children.') assert isinstance(prop.value, str) if prop.value.endswith('.vpk'): fsys = VPKFileSystem(str((game_root / prop.value).resolve())) else: fsys = RawFileSystem(str((game_root / prop.value).resolve())) if prop.name in ('prefix', 'priority'): fsys_chain.add_sys(fsys, priority=True) elif prop.name == 'nopack': blacklist.add(fsys) elif prop.name in ('path', 'pack'): fsys_chain.add_sys(fsys) else: raise ValueError( 'Unknown searchpath ' 'key "{}"!'.format(prop.real_name) ) plugins = set() # type: Set[Plugin] # find all the plugins and make plugin objects out of them for prop in conf.get(Property, 'plugins'): # type: Property if prop.has_children(): raise ValueError('Config "plugins" value cannot have children.') assert isinstance(prop.value, str) path = (game_root / Path(prop.value)).resolve() if prop.name in ("path", "recursive"): if not path.is_dir(): raise ValueError("'{}' is not a directory".format(path)) # want to recursive glob if key is recursive pattern = "*.py" if prop.name == "path" else "**/*.py" #find all .py files, make Plugins for p in path.glob(pattern): plugins.add(Plugin(path / p)) elif prop.name == "single": plugins.add(Plugin(path)) else: raise ValueError("Unknown plugins key {}".format(prop.real_name)) return conf, game, fsys_chain, blacklist, plugins
def decompile_model( fsys: FileSystemChain, cache_loc: Path, crowbar: Path, filename: str, checksum: bytes, ) -> Optional[QC]: """Use Crowbar to decompile models directly for propcombining.""" cache_folder = cache_loc / Path(filename).with_suffix('') info_path = cache_folder / 'info.kv' if cache_folder.exists(): try: with info_path.open() as f: cache_props = Property.parse(f).find_key('qc', []) except (FileNotFoundError, KeyValError): pass else: # Previous compilation. if checksum == bytes.fromhex(cache_props['checksum', '']): ref_smd = cache_props['ref', ''] if not ref_smd: return None phy_smd = cache_props['phy', None] if phy_smd is not None: phy_smd = str(cache_folder / phy_smd) return QC( str(info_path), str(cache_folder / ref_smd), phy_smd, cache_props.float('ref_scale', 1.0), cache_props.float('phy_scale', 1.0), ) # Otherwise, re-decompile. LOGGER.info('Decompiling {}...', filename) qc: Optional[QC] = None # Extract out the model to a temp dir. with TemporaryDirectory() as tempdir, fsys: stem = Path(filename).stem filename_no_ext = filename[:-4] for mdl_ext in MDL_EXTS: try: file = fsys[filename_no_ext + mdl_ext] except FileNotFoundError: pass else: with file.open_bin() as src, Path(tempdir, stem + mdl_ext).open('wb') as dest: shutil.copyfileobj(src, dest) LOGGER.debug('Extracted "{}" to "{}"', filename, tempdir) args = [ str(crowbar), 'decompile', '-i', str(Path(tempdir, stem + '.mdl')), '-o', str(cache_folder), ] LOGGER.debug('Executing {}', ' '.join(args)) result = subprocess.run(args) if result.returncode != 0: LOGGER.warning('Could not decompile "{}"!', filename) return None # There should now be a QC file here. for qc_path in cache_folder.glob('*.qc'): qc_result = parse_qc(cache_folder, qc_path) break else: # not found. LOGGER.warning('No QC outputted into {}', cache_folder) qc_result = None qc_path = Path() cache_props = Property('qc', []) cache_props['checksum'] = checksum.hex() if qc_result is not None: ( model_name, ref_scale, ref_smd, phy_scale, phy_smd, ) = qc_result qc = QC( str(qc_path).replace('\\', '/'), str(ref_smd).replace('\\', '/'), str(phy_smd).replace('\\', '/') if phy_smd else None, ref_scale, phy_scale, ) cache_props['ref'] = Path(ref_smd).name cache_props['ref_scale'] = format(ref_scale, '.6g') if phy_smd is not None: cache_props['phy'] = Path(phy_smd).name cache_props['phy_scale'] = format(phy_scale, '.6g') else: cache_props['ref'] = '' # Mark as not present. with info_path.open('w') as f: for line in cache_props.export(): f.write(line) return qc
def _parse_template(loc: UnparsedTemplate) -> Template: """Parse a template VMF.""" filesys: FileSystem if os.path.isdir(loc.pak_path): filesys = RawFileSystem(loc.pak_path) else: ext = os.path.splitext(loc.pak_path)[1].casefold() if ext in ('.bee_pack', '.zip'): filesys = ZipFileSystem(loc.pak_path) elif ext == '.vpk': filesys = VPKFileSystem(loc.pak_path) else: raise ValueError(f'Unknown filesystem type for "{loc.pak_path}"!') with filesys[loc.path].open_str() as f: props = Property.parse(f, f'{loc.pak_path}:{loc.path}') vmf = srctools.VMF.parse(props, preserve_ids=True) del props, filesys, f # Discard all this data. # visgroup -> list of brushes/overlays detail_ents: dict[str, list[Solid]] = defaultdict(list) world_ents: dict[str, list[Solid]] = defaultdict(list) overlay_ents: dict[str, list[Entity]] = defaultdict(list) color_pickers: list[ColorPicker] = [] tile_setters: list[TileSetter] = [] voxel_setters: list[VoxelSetter] = [] conf_ents = vmf.by_class['bee2_template_conf'] if len(conf_ents) > 1: raise ValueError( f'Multiple configuration entities in template "{loc.id}"!') elif not conf_ents: raise ValueError(f'No configration entity for template "{loc.id}"!') else: [conf] = conf_ents if conf['template_id'].upper() != loc.id: raise ValueError( f'Mismatch in template IDs for {conf["template_id"]} and {loc.id}') def yield_world_detail() -> Iterator[tuple[list[Solid], bool, set[str]]]: """Yield all world/detail solids in the map. This also indicates if it's a func_detail, and the visgroup IDs. (Those are stored in the ent for detail, and the solid for world.) """ for brush in vmf.brushes: yield [brush], False, brush.visgroup_ids for detail in vmf.by_class['func_detail']: yield detail.solids, True, detail.visgroup_ids force = conf['temp_type'] force_is_detail: Optional[bool] if force.casefold() == 'detail': force_is_detail = True elif force.casefold() == 'world': force_is_detail = False else: force_is_detail = None visgroup_names = {vis.id: vis.name.casefold() for vis in vmf.vis_tree} conf_auto_visgroup = 1 if srctools.conv_bool( conf['detail_auto_visgroup']) else 0 if not srctools.conv_bool(conf['discard_brushes']): for brushes, is_detail, vis_ids in yield_world_detail(): visgroups = list(map(visgroup_names.__getitem__, vis_ids)) if len(visgroups) > 1: raise ValueError('Template "{}" has brush with two ' 'visgroups! ({})'.format( loc.id, ', '.join(visgroups))) # No visgroup = '' visgroup = visgroups[0] if visgroups else '' # Auto-visgroup puts func_detail ents in unique visgroups. if is_detail and not visgroup and conf_auto_visgroup: visgroup = '__auto_group_{}__'.format(conf_auto_visgroup) # Reuse as the unique index, >0 are True too.. conf_auto_visgroup += 1 # Check this after auto-visgroup, so world/detail can be used to # opt into the grouping, then overridden to be the same. if force_is_detail is not None: is_detail = force_is_detail if is_detail: detail_ents[visgroup].extend(brushes) else: world_ents[visgroup].extend(brushes) for ent in vmf.by_class['info_overlay']: visgroups = list(map(visgroup_names.__getitem__, ent.visgroup_ids)) if len(visgroups) > 1: raise ValueError('Template "{}" has overlay with two ' 'visgroups! ({})'.format(loc.id, ', '.join(visgroups))) # No visgroup = '' overlay_ents[visgroups[0] if visgroups else ''].append(ent) for ent in vmf.by_class['bee2_template_colorpicker']: # Parse the colorpicker data. try: priority = Decimal(ent['priority']) except ArithmeticError: LOGGER.warning( 'Bad priority for colorpicker in "{}" template!', loc.id, ) priority = Decimal(0) try: remove_after = AfterPickMode(ent['remove_brush', '0']) except ValueError: LOGGER.warning( 'Bad remove-brush mode for colorpicker in "{}" template!', loc.id, ) remove_after = AfterPickMode.NONE color_pickers.append( ColorPicker( priority=priority, name=ent['targetname'], visgroups=set(map(visgroup_names.__getitem__, ent.visgroup_ids)), offset=Vec.from_str(ent['origin']), normal=Vec(x=1) @ Angle.from_str(ent['angles']), sides=ent['faces'].split(' '), grid_snap=srctools.conv_bool(ent['grid_snap']), after=remove_after, use_pattern=srctools.conv_bool(ent['use_pattern']), force_tex_white=ent['tex_white'], force_tex_black=ent['tex_black'], )) for ent in vmf.by_class['bee2_template_voxelsetter']: tile_type = TILE_SETTER_SKINS[srctools.conv_int(ent['skin'])] voxel_setters.append( VoxelSetter( offset=Vec.from_str(ent['origin']), normal=Vec(z=1) @ Angle.from_str(ent['angles']), visgroups=set(map(visgroup_names.__getitem__, ent.visgroup_ids)), tile_type=tile_type, force=srctools.conv_bool(ent['force']), )) for ent in vmf.by_class['bee2_template_tilesetter']: tile_type = TILE_SETTER_SKINS[srctools.conv_int(ent['skin'])] color = ent['color'] if color == 'tile': try: color = tile_type.color except ValueError: # Non-tile types. color = None elif color == 'invert': color = 'INVERT' elif color == 'match': color = None elif color != 'copy': raise ValueError('Invalid TileSetter color ' '"{}" for "{}"'.format(color, loc.id)) tile_setters.append( TileSetter( offset=Vec.from_str(ent['origin']), normal=Vec(z=1) @ Angle.from_str(ent['angles']), visgroups=set(map(visgroup_names.__getitem__, ent.visgroup_ids)), color=color, tile_type=tile_type, picker_name=ent['color_picker'], force=srctools.conv_bool(ent['force']), )) coll: list[CollisionDef] = [] for ent in vmf.by_class['bee2_collision_bbox']: visgroups = set(map(visgroup_names.__getitem__, ent.visgroup_ids)) for bbox in collisions.BBox.from_ent(ent): coll.append(CollisionDef(bbox, visgroups)) return Template( loc.id, set(visgroup_names.values()), world_ents, detail_ents, overlay_ents, conf['skip_faces'].split(), conf['realign_faces'].split(), conf['overlay_faces'].split(), conf['vertical_faces'].split(), color_pickers, tile_setters, voxel_setters, coll, )
def parse( path: Path ) -> Tuple[Config, Game, FileSystemChain, Set[FileSystem], PluginFinder, ]: """From some directory, locate and parse the config file. This then constructs and customises each object according to config options. The first srctools.vdf file found in a parent directory is parsed. If none can be found, it tries to find the first subfolder of 'common/' and writes a default copy there. FileNotFoundError is raised if none can be found. This returns: * The config. * Parsed gameinfo. * The chain of filesystems. * A packing blacklist. * The plugin loader. """ conf = Config(OPTIONS) # If the path is a folder, add a dummy folder so parents yields it. # That way we check for a config in this folder. if not path.suffix: path /= 'unused' for folder in path.parents: conf_path = folder / CONF_NAME if conf_path.exists(): LOGGER.info('Config path: "{}"', conf_path.absolute()) with open(conf_path) as f: props = Property.parse(f, conf_path) conf.path = conf_path conf.load(props) break else: LOGGER.warning('Cannot find a valid config file!') # Apply all the defaults. conf.load(Property(None, [])) # Try to write out a default file in the game folder. for folder in path.parents: if folder.parent.stem == 'common': break else: # Give up, write to working directory. folder = Path() conf.path = folder / CONF_NAME LOGGER.warning('Writing default to "{}"', conf.path) with AtomicWriter(str(conf.path)) as f: conf.save(f) game = Game((folder / conf.get(str, 'gameinfo')).resolve()) fsys_chain = game.get_filesystem() blacklist = set() # type: Set[FileSystem] if not conf.get(bool, 'pack_vpk'): for fsys, prefix in fsys_chain.systems: if isinstance(fsys, VPKFileSystem): blacklist.add(fsys) game_root = game.root for prop in conf.get(Property, 'searchpaths'): # type: Property if prop.has_children(): raise ValueError( 'Config "searchpaths" value cannot have children.') assert isinstance(prop.value, str) if prop.value.endswith('.vpk'): fsys = VPKFileSystem(str((game_root / prop.value).resolve())) else: fsys = RawFileSystem(str((game_root / prop.value).resolve())) if prop.name in ('prefix', 'priority'): fsys_chain.add_sys(fsys, priority=True) elif prop.name == 'nopack': blacklist.add(fsys) elif prop.name in ('path', 'pack'): fsys_chain.add_sys(fsys) else: raise ValueError('Unknown searchpath ' 'key "{}"!'.format(prop.real_name)) sources: Dict[Path, PluginSource] = {} builtin_transforms = (Path(sys.argv[0]).parent / 'transforms').resolve() # find all the plugins and make plugin objects out of them for prop in conf.get(Property, 'plugins'): if prop.has_children(): raise ValueError('Config "plugins" value cannot have children.') assert isinstance(prop.value, str) path = (game_root / Path(prop.value)).resolve() if prop.name in ('path', "recursive", 'folder'): if not path.is_dir(): raise ValueError("'{}' is not a directory".format(path)) is_recursive = prop.name == "recursive" try: source = sources[path] except KeyError: sources[path] = PluginSource(path, is_recursive) else: if is_recursive and not source.recursive: # Upgrade to recursive. source.recursive = True elif prop.name in ('single', 'file'): parent = path.parent try: source = sources[parent] except KeyError: source = sources[parent] = PluginSource(parent, False) source.autoload_files.add(path) elif prop.name == "_builtin_": # For development purposes, redirect builtin folder. builtin_transforms = path else: raise ValueError("Unknown plugins key {}".format(prop.real_name)) for source in sources.values(): LOGGER.debug('Plugin path: "{}", recursive={}, files={}', source.folder, source.recursive, sorted(source.autoload_files)) LOGGER.debug('Builtin plugin path is {}', builtin_transforms) if builtin_transforms not in sources: sources[builtin_transforms] = PluginSource(builtin_transforms, True) plugin_finder = PluginFinder('srctools.bsp_transforms.plugin', sources.values()) sys.meta_path.append(plugin_finder) return conf, game, fsys_chain, blacklist, plugin_finder
def main(args: List[str]) -> None: """Main script.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "-f", "--filter", help="filter output to only display resources in this subfolder. " "This can be used multiple times.", type=str.casefold, action='append', metavar='folder', dest='filters', ) parser.add_argument( "-u", "--unused", help="Instead of showing depenencies, show files in the filtered " "folders that are unused.", action='store_true', ) parser.add_argument( "game", help="either location of a gameinfo.txt file, or any root folder.", ) parser.add_argument( "path", help="the files to load. The path can have a single * in the " "filename to match files with specific extensions and a prefix.", ) result = parser.parse_args(args) if result.unused and not result.filters: raise ValueError( 'At least one filter must be provided in "unused" mode.') if result.game: try: fsys = Game(result.game).get_filesystem() except FileNotFoundError: fsys = FileSystemChain(RawFileSystem(result.game)) else: fsys = FileSystemChain() packlist = PackList(fsys) file_path: str = result.path print('Finding files...') with fsys: if '*' in file_path: # Multiple files if file_path.count('*') > 1: raise ValueError('Multiple * in path!') prefix, suffix = file_path.split('*') folder, prefix = os.path.split(prefix) prefix = prefix.casefold() suffix = suffix.casefold() print(f'Prefix: {prefix!r}, suffix: {suffix!r}') print(f'Searching folder {folder}...') files = [] for file in fsys.walk_folder(folder): file_path = file.path.casefold() if not os.path.basename(file_path).startswith(prefix): continue if file_path.endswith(suffix): print(' ' + file.path) files.append(file) else: # Single file files = [fsys[file_path]] for file in files: ext = file.path[-4:].casefold() if ext == '.vmf': with file.open_str() as f: vmf_props = Property.parse(f) vmf = VMF.parse(vmf_props) packlist.pack_fgd(vmf, fgd) del vmf, vmf_props # Hefty, don't want to keep. elif ext == '.bsp': child_sys = fsys.get_system(file) if not isinstance(child_sys, RawFileSystem): raise ValueError('Cannot inspect BSPs in VPKs!') bsp = BSP(os.path.join(child_sys.path, file.path)) packlist.pack_from_bsp(bsp) packlist.pack_fgd(bsp.read_ent_data(), fgd) del bsp else: packlist.pack_file(file.path) print('Evaluating dependencies...') packlist.eval_dependencies() print('Done.') if result.unused: print('Unused files:') used = set(packlist.filenames()) for folder in result.filters: for file in fsys.walk_folder(folder): if file.path.casefold() not in used: print(' ' + file.path) else: print('Dependencies:') for filename in packlist.filenames(): if not result.filters or any( map(filename.casefold().startswith, result.filters)): print(' ' + filename)
def load_templates() -> None: """Load in the template file, used for import_template().""" with open(TEMPLATE_LOCATION) as file: props = Property.parse(file, TEMPLATE_LOCATION) vmf = srctools.VMF.parse(props, preserve_ids=True) def make_subdict() -> Dict[str, list]: return defaultdict(list) # detail_ents[temp_id][visgroup] detail_ents = defaultdict( make_subdict) # type: Dict[str, Dict[str, List[Solid]]] world_ents = defaultdict( make_subdict) # type: Dict[str, Dict[str, List[Solid]]] overlay_ents = defaultdict( make_subdict) # type: Dict[str, Dict[str, List[Entity]]] conf_ents = {} color_pickers = defaultdict(list) # type: Dict[str, List[ColorPicker]] tile_setters = defaultdict(list) # type: Dict[str, List[TileSetter]] for ent in vmf.by_class['bee2_template_world']: world_ents[ent['template_id'].casefold()][ ent['visgroup'].casefold()].extend(ent.solids) for ent in vmf.by_class['bee2_template_detail']: detail_ents[ent['template_id'].casefold()][ ent['visgroup'].casefold()].extend(ent.solids) for ent in vmf.by_class['bee2_template_overlay']: overlay_ents[ent['template_id'].casefold()][ ent['visgroup'].casefold()].append(ent) for ent in vmf.by_class['bee2_template_conf']: conf_ents[ent['template_id'].casefold()] = ent for ent in vmf.by_class['bee2_template_scaling']: temp = ScalingTemplate.parse(ent) _TEMPLATES[temp.id.casefold()] = temp for ent in vmf.by_class['bee2_template_colorpicker']: # Parse the colorpicker data. temp_id = ent['template_id'].casefold() try: priority = Decimal(ent['priority']) except ValueError: LOGGER.warning( 'Bad priority for colorpicker in "{}" template!', temp_id.upper(), ) priority = Decimal(0) try: remove_after = AfterPickMode(ent['remove_brush', '0']) except ValueError: LOGGER.warning( 'Bad remove-brush mode for colorpicker in "{}" template!', temp_id.upper(), ) remove_after = AfterPickMode.NONE color_pickers[temp_id].append( ColorPicker( priority, name=ent['targetname'], visgroups=set(ent['visgroups'].split(' ')) - {''}, offset=Vec.from_str(ent['origin']), normal=Vec(x=1) @ Angle.from_str(ent['angles']), sides=ent['faces'].split(' '), grid_snap=srctools.conv_bool(ent['grid_snap']), after=remove_after, use_pattern=srctools.conv_bool(ent['use_pattern']), force_tex_white=ent['tex_white'], force_tex_black=ent['tex_black'], )) for ent in vmf.by_class['bee2_template_tilesetter']: # Parse the tile setter data. temp_id = ent['template_id'].casefold() tile_type = TILE_SETTER_SKINS[srctools.conv_int(ent['skin'])] color = ent['color'] if color == 'tile': try: color = tile_type.color except ValueError: # Non-tile types. color = None elif color == 'invert': color = 'INVERT' elif color == 'match': color = None elif color != 'copy': raise ValueError('Invalid TileSetter color ' '"{}" for "{}"'.format(color, temp_id)) tile_setters[temp_id].append( TileSetter( offset=Vec.from_str(ent['origin']), normal=Vec(z=1) @ Angle.from_str(ent['angles']), visgroups=set(ent['visgroups'].split(' ')) - {''}, color=color, tile_type=tile_type, picker_name=ent['color_picker'], force=srctools.conv_bool(ent['force']), )) temp_ids = set(conf_ents).union( detail_ents, world_ents, overlay_ents, color_pickers, tile_setters, ) for temp_id in temp_ids: try: conf = conf_ents[temp_id] except KeyError: overlay_faces = [] # type: List[str] skip_faces = [] # type: List[str] vertical_faces = [] # type: List[str] realign_faces = [] # type: List[str] else: vertical_faces = conf['vertical_faces'].split() realign_faces = conf['realign_faces'].split() overlay_faces = conf['overlay_faces'].split() skip_faces = conf['skip_faces'].split() _TEMPLATES[temp_id.casefold()] = Template( temp_id, world_ents[temp_id], detail_ents[temp_id], overlay_ents[temp_id], skip_faces, realign_faces, overlay_faces, vertical_faces, color_pickers[temp_id], tile_setters[temp_id], )
def load_templates(): """Load in the template file, used for import_template().""" with open(TEMPLATE_LOCATION) as file: props = Property.parse(file, TEMPLATE_LOCATION) vmf = srctools.VMF.parse(props, preserve_ids=True) def make_subdict(): return defaultdict(list) # detail_ents[temp_id][visgroup] detail_ents = defaultdict(make_subdict) world_ents = defaultdict(make_subdict) overlay_ents = defaultdict(make_subdict) conf_ents = {} color_pickers = defaultdict(list) for ent in vmf.by_class['bee2_template_world']: world_ents[ ent['template_id'].casefold() ][ ent['visgroup'].casefold() ].extend(ent.solids) for ent in vmf.by_class['bee2_template_detail']: detail_ents[ ent['template_id'].casefold() ][ ent['visgroup'].casefold() ].extend(ent.solids) for ent in vmf.by_class['bee2_template_overlay']: overlay_ents[ ent['template_id'].casefold() ][ ent['visgroup'].casefold() ].append(ent) for ent in vmf.by_class['bee2_template_conf']: conf_ents[ent['template_id'].casefold()] = ent for ent in vmf.by_class['bee2_template_scaling']: temp = ScalingTemplate.parse(ent) TEMPLATES[temp.id.casefold()] = temp for ent in vmf.by_class['bee2_template_colorpicker']: # Parse the colorpicker data. temp_id = ent['template_id'].casefold() try: priority = Decimal(ent['priority']) except ValueError: LOGGER.warning( 'Bad priority for colorpicker in "{}" template!', temp_id.upper(), ) priority = Decimal(0) color_pickers[temp_id].append(ColorPicker( priority, offset=Vec.from_str(ent['origin']), normal=Vec(x=1).rotate_by_str(ent['angles']), sides=ent['faces'].split(' '), grid_snap=srctools.conv_bool(ent['grid_snap']), remove_brush=srctools.conv_bool(ent['remove_brush']), )) for temp_id in set(detail_ents).union(world_ents, overlay_ents): try: conf = conf_ents[temp_id] except KeyError: overlay_faces = [] skip_faces = [] vertical_faces = [] realign_faces = [] else: vertical_faces = conf['vertical_faces'].split() realign_faces = conf['realign_faces'].split() overlay_faces = conf['overlay_faces'].split() skip_faces = conf['skip_faces'].split() TEMPLATES[temp_id.casefold()] = Template( temp_id, world_ents[temp_id], detail_ents[temp_id], overlay_ents[temp_id], skip_faces, realign_faces, overlay_faces, vertical_faces, color_pickers[temp_id], )