def save(self, ignore_readonly=False): """Save the palette file into the specified location. If ignore_readonly is true, this will ignore the `prevent_overwrite` property of the palette (allowing resaving those properties over old versions). Otherwise those palettes always create a new file. """ LOGGER.info('Saving "{}"!', self.name) props = Property(None, [ Property('Name', self.name), Property('TransName', self.trans_name), Property('ReadOnly', srctools.bool_as_int(self.prevent_overwrite)), Property('Items', [ Property(item_id, str(subitem)) for item_id, subitem in self.pos ]) ]) # If default, don't include in the palette file. # Remove the translated name, in case it's not going to write # properly to the file. if self.trans_name: props['Name'] = '' else: del props['TransName'] if not self.prevent_overwrite: del props['ReadOnly'] if self.settings is not None: self.settings.name = 'Settings' props.append(self.settings.copy()) # We need to write a new file, determine a valid path. # Use a hash to ensure it's a valid path (without '-' if negative) # If a conflict occurs, add ' ' and hash again to get a different # value. if self.filename is None or (self.prevent_overwrite and not ignore_readonly): hash_src = self.name while True: hash_filename = str(abs(hash(hash_src))) + PAL_EXT if os.path.isfile(hash_filename): # Add a random character to iterate the hash. hash_src += chr(random.randrange(0x10ffff)) else: file = open(os.path.join(PAL_DIR, hash_filename), 'w', encoding='utf8') self.filename = os.path.join(PAL_DIR, hash_filename) break else: file = open(os.path.join(PAL_DIR, self.filename), 'w', encoding='utf8') with file: for line in props.export(): file.write(line)
def gen_sound_manifest(additional, excludes): """Generate a new game_sounds_manifest.txt file. This includes all the current scripts defined, plus any custom ones. Excludes is a list of scripts to remove from the listing - this allows overriding the sounds without VPK overrides. """ if not additional: return # Don't pack, there aren't any new sounds.. orig_manifest = os.path.join( '..', SOUND_MAN_FOLDER.get(CONF['game_id', ''], 'portal2'), 'scripts', 'game_sounds_manifest.txt', ) try: with open(orig_manifest) as f: props = Property.parse(f, orig_manifest).find_key( 'game_sounds_manifest', [], ) except FileNotFoundError: # Assume no sounds props = Property('game_sounds_manifest', []) scripts = [prop.value for prop in props.find_all('precache_file')] for script in additional: scripts.append(script) for script in excludes: try: scripts.remove(script) except ValueError: LOGGER.warning( '"{}" should be excluded, but it\'s' ' not in the manifest already!', script, ) # Build and unbuild it to strip other things out - Valve includes a bogus # 'new_sound_scripts_must_go_below_here' entry.. new_props = Property('game_sounds_manifest', [Property('precache_file', file) for file in scripts]) inject_loc = os.path.join('bee2', 'inject', 'soundscript_manifest.txt') with open(inject_loc, 'w') as f: for line in new_props.export(): f.write(line) LOGGER.info('Written new soundscripts_manifest..')
def gen_part_manifest(additional): """Generate a new particle system manifest file. This includes all the current ones defined, plus any custom ones. """ if not additional: return # Don't pack, there aren't any new particles.. orig_manifest = os.path.join( '..', GAME_FOLDER.get(CONF['game_id', ''], 'portal2'), 'particles', 'particles_manifest.txt', ) try: with open(orig_manifest) as f: props = Property.parse(f, orig_manifest).find_key( 'particles_manifest', [], ) except FileNotFoundError: # Assume no particles props = Property('particles_manifest', []) parts = [prop.value for prop in props.find_all('file')] for particle in additional: parts.append(particle) # Build and unbuild it to strip comments and similar lines. new_props = Property('particles_manifest', [Property('file', file) for file in parts]) inject_loc = os.path.join('bee2', 'inject', 'particles_manifest.txt') with open(inject_loc, 'w') as f: for line in new_props.export(): f.write(line) LOGGER.info('Written new particles_manifest..')
def export(exp_data: ExportData) -> None: """Export all the packlists.""" pack_block = Property('PackList', []) for pack in PackList.all(): # type: PackList # Build a # "Pack_id" # { # "File" "filename" # "File" "filename" # } # block for each packlist files = [Property('File', file) for file in pack.files] pack_block.append(Property( pack.id, files, )) LOGGER.info('Writing packing list!') with open(exp_data.game.abs_path('bin/bee2/pack_list.cfg'), 'w') as pack_file: for line in pack_block.export(): pack_file.write(line)
def build_instance_data(editoritems: Property): """Build a property tree listing all of the instances for each item. as well as another listing the input and output commands. VBSP uses this to reduce duplication in VBSP_config files. This additionally strips custom instance definitions from the original list. """ instance_locs = Property("AllInstances", []) cust_inst = Property("CustInstances", []) commands = Property("Connections", []) item_classes = Property("ItemClasses", []) root_block = Property(None, [ instance_locs, item_classes, cust_inst, commands, ]) for item in editoritems.find_all("Item"): instance_block = Property(item['Type'], []) instance_locs.append(instance_block) comm_block = Property(item['Type'], []) for inst_block in item.find_all("Exporting", "instances"): for inst in inst_block.value[:]: # type: Property if inst.name.isdigit(): # Direct Portal 2 value instance_block.append( Property('Instance', inst['Name']) ) else: # It's a custom definition, remove from editoritems inst_block.value.remove(inst) # Allow the name to start with 'bee2_' also to match # the <> definitions - it's ignored though. name = inst.name if name[:5] == 'bee2_': name = name[5:] cust_inst.set_key( (item['type'], name), # Allow using either the normal block format, # or just providing the file - we don't use the # other values. inst['name'] if inst.has_children() else inst.value, ) # Look in the Inputs and Outputs blocks to find the io definitions. # Copy them to property names like 'Input_Activate'. for io_type in ('Inputs', 'Outputs'): for block in item.find_all('Exporting', io_type, CONN_NORM): for io_prop in block: comm_block[ io_type[:-1] + '_' + io_prop.real_name ] = io_prop.value # The funnel item type is special, having the additional input type. # Handle that specially. if item['type'].casefold() == 'item_tbeam': for block in item.find_all('Exporting', 'Inputs', CONN_FUNNEL): for io_prop in block: comm_block['TBeam_' + io_prop.real_name] = io_prop.value # Fizzlers don't work correctly with outputs. This is a signal to # conditions.fizzler, but it must be removed in editoritems. if item['ItemClass', ''].casefold() == 'itembarrierhazard': for block in item.find_all('Exporting', 'Outputs'): if CONN_NORM in block: del block[CONN_NORM] # Record the itemClass for each item type. item_classes[item['type']] = item['ItemClass', 'ItemBase'] # Only add the block if the item actually has IO. if comm_block.value: commands.append(comm_block) return root_block.export()
def decompile_model( fsys: FileSystemChain, cache_loc: Path, crowbar: Path, filename: str, checksum: bytes, ) -> Optional[QC]: """Use Crowbar to decompile models directly for propcombining.""" cache_folder = cache_loc / Path(filename).with_suffix('') info_path = cache_folder / 'info.kv' if cache_folder.exists(): try: with info_path.open() as f: cache_props = Property.parse(f).find_key('qc', []) except (FileNotFoundError, KeyValError): pass else: # Previous compilation. if checksum == bytes.fromhex(cache_props['checksum', '']): ref_smd = cache_props['ref', ''] if not ref_smd: return None phy_smd = cache_props['phy', None] if phy_smd is not None: phy_smd = str(cache_folder / phy_smd) return QC( str(info_path), str(cache_folder / ref_smd), phy_smd, cache_props.float('ref_scale', 1.0), cache_props.float('phy_scale', 1.0), ) # Otherwise, re-decompile. LOGGER.info('Decompiling {}...', filename) qc: Optional[QC] = None # Extract out the model to a temp dir. with TemporaryDirectory() as tempdir, fsys: stem = Path(filename).stem filename_no_ext = filename[:-4] for mdl_ext in MDL_EXTS: try: file = fsys[filename_no_ext + mdl_ext] except FileNotFoundError: pass else: with file.open_bin() as src, Path(tempdir, stem + mdl_ext).open('wb') as dest: shutil.copyfileobj(src, dest) LOGGER.debug('Extracted "{}" to "{}"', filename, tempdir) args = [ str(crowbar), 'decompile', '-i', str(Path(tempdir, stem + '.mdl')), '-o', str(cache_folder), ] LOGGER.debug('Executing {}', ' '.join(args)) result = subprocess.run(args) if result.returncode != 0: LOGGER.warning('Could not decompile "{}"!', filename) return None # There should now be a QC file here. for qc_path in cache_folder.glob('*.qc'): qc_result = parse_qc(cache_folder, qc_path) break else: # not found. LOGGER.warning('No QC outputted into {}', cache_folder) qc_result = None qc_path = Path() cache_props = Property('qc', []) cache_props['checksum'] = checksum.hex() if qc_result is not None: ( model_name, ref_scale, ref_smd, phy_scale, phy_smd, ) = qc_result qc = QC( str(qc_path).replace('\\', '/'), str(ref_smd).replace('\\', '/'), str(phy_smd).replace('\\', '/') if phy_smd else None, ref_scale, phy_scale, ) cache_props['ref'] = Path(ref_smd).name cache_props['ref_scale'] = format(ref_scale, '.6g') if phy_smd is not None: cache_props['phy'] = Path(phy_smd).name cache_props['phy_scale'] = format(phy_scale, '.6g') else: cache_props['ref'] = '' # Mark as not present. with info_path.open('w') as f: for line in cache_props.export(): f.write(line) return qc
def build_instance_data(editoritems: Property): """Build a property tree listing all of the instances for each item. as well as another listing the input and output commands. VBSP uses this to reduce duplication in VBSP_config files. This additionally strips custom instance definitions from the original list. """ instance_locs = Property("AllInstances", []) cust_inst = Property("CustInstances", []) commands = Property("Connections", []) item_classes = Property("ItemClasses", []) root_block = Property(None, [instance_locs, item_classes, cust_inst, commands]) for item in editoritems.find_all("Item"): instance_block = Property(item["Type"], []) instance_locs.append(instance_block) comm_block = Property(item["Type"], []) for inst_block in item.find_all("Exporting", "instances"): for inst in inst_block.value[:]: # type: Property if inst.name.isdigit(): # Direct Portal 2 value instance_block.append(Property("Instance", inst["Name"])) else: # It's a custom definition, remove from editoritems inst_block.value.remove(inst) # Allow the name to start with 'bee2_' also to match # the <> definitions - it's ignored though. name = inst.name if name[:5] == "bee2_": name = name[5:] cust_inst.set_key( (item["type"], name), # Allow using either the normal block format, # or just providing the file - we don't use the # other values. inst["name"] if inst.has_children() else inst.value, ) # Look in the Inputs and Outputs blocks to find the io definitions. # Copy them to property names like 'Input_Activate'. for io_type in ("Inputs", "Outputs"): for block in item.find_all("Exporting", io_type, CONN_NORM): for io_prop in block: comm_block[io_type[:-1] + "_" + io_prop.real_name] = io_prop.value # The funnel item type is special, having the additional input type. # Handle that specially. if item["type"] == "item_tbeam": for block in item.find_all("Exporting", "Inputs", CONN_FUNNEL): for io_prop in block: comm_block["TBEAM_" + io_prop.real_name] = io_prop.value # Fizzlers don't work correctly with outputs. This is a signal to # conditions.fizzler, but it must be removed in editoritems. if item["ItemClass", ""].casefold() == "itembarrierhazard": for block in item.find_all("Exporting", "Outputs"): if CONN_NORM in block: del block[CONN_NORM] # Record the itemClass for each item type. item_classes[item["type"]] = item["ItemClass", "ItemBase"] # Only add the block if the item actually has IO. if comm_block.value: commands.append(comm_block) return root_block.export()
def export( self, style: packages.Style, selected_objects: dict, should_refresh=False, ) -> Tuple[bool, bool]: """Generate the editoritems.txt and vbsp_config. - If no backup is present, the original editoritems is backed up. - For each object type, run its .export() function with the given - item. - Styles are a special case. """ LOGGER.info('-' * 20) LOGGER.info('Exporting Items and Style for "{}"!', self.name) LOGGER.info('Style = {}', style.id) for obj, selected in selected_objects.items(): # Skip the massive dict in items if obj == 'Item': selected = selected[0] LOGGER.info('{} = {}', obj, selected) # VBSP, VRAD, editoritems export_screen.set_length('BACK', len(FILES_TO_BACKUP)) # files in compiler/ try: num_compiler_files = sum( 1 for file in utils.install_path('compiler').rglob('*')) except FileNotFoundError: num_compiler_files = 0 if self.steamID == utils.STEAM_IDS['APERTURE TAG']: # Coop paint gun instance num_compiler_files += 1 if num_compiler_files == 0: LOGGER.warning('No compiler files!') export_screen.skip_stage('COMP') else: export_screen.set_length('COMP', num_compiler_files) LOGGER.info('Should refresh: {}', should_refresh) if should_refresh: # Check to ensure the cache needs to be copied over.. should_refresh = self.cache_invalid() if should_refresh: LOGGER.info("Cache invalid - copying..") else: LOGGER.info("Skipped copying cache!") # Each object type # Editoritems # VBSP_config # Instance list # Editor models. # FGD file # Gameinfo export_screen.set_length('EXP', len(packages.OBJ_TYPES) + 6) # Do this before setting music and resources, # those can take time to compute. export_screen.show() try: if should_refresh: # Count the files. export_screen.set_length( 'RES', sum(1 for file in res_system.walk_folder_repeat()), ) else: export_screen.skip_stage('RES') export_screen.skip_stage('MUS') # Make the folders we need to copy files to, if desired. os.makedirs(self.abs_path('bin/bee2/'), exist_ok=True) # Start off with the style's data. vbsp_config = Property(None, []) vbsp_config += style.config.copy() all_items = style.items.copy() renderables = style.renderables.copy() export_screen.step('EXP') vpk_success = True # Export each object type. for obj_name, obj_data in packages.OBJ_TYPES.items(): if obj_name == 'Style': continue # Done above already LOGGER.info('Exporting "{}"', obj_name) selected = selected_objects.get(obj_name, None) try: obj_data.cls.export( packages.ExportData( game=self, selected=selected, all_items=all_items, renderables=renderables, vbsp_conf=vbsp_config, selected_style=style, )) except packages.NoVPKExport: # Raised by StyleVPK to indicate it failed to copy. vpk_success = False export_screen.step('EXP') vbsp_config.set_key(('Options', 'Game_ID'), self.steamID) vbsp_config.set_key( ('Options', 'dev_mode'), srctools.bool_as_int(optionWindow.DEV_MODE.get())) # If there are multiple of these blocks, merge them together. # They will end up in this order. vbsp_config.merge_children( 'Textures', 'Fizzlers', 'Options', 'StyleVars', 'DropperItems', 'Conditions', 'Quotes', 'PackTriggers', ) for name, file, ext in FILES_TO_BACKUP: item_path = self.abs_path(file + ext) backup_path = self.abs_path(file + '_original' + ext) if not os.path.isfile(item_path): # We can't backup at all. should_backup = False elif name == 'Editoritems': should_backup = not os.path.isfile(backup_path) else: # Always backup the non-_original file, it'd be newer. # But only if it's Valves - not our own. should_backup = should_backup_app(item_path) backup_is_good = should_backup_app(backup_path) LOGGER.info( '{}{}: normal={}, backup={}', file, ext, 'Valve' if should_backup else 'BEE2', 'Valve' if backup_is_good else 'BEE2', ) if not should_backup and not backup_is_good: # It's a BEE2 application, we have a problem. # Both the real and backup are bad, we need to get a # new one. try: os.remove(backup_path) except FileNotFoundError: pass try: os.remove(item_path) except FileNotFoundError: pass export_screen.reset() if messagebox.askokcancel( title=_('BEE2 - Export Failed!'), message=_( 'Compiler file {file} missing. ' 'Exit Steam applications, then press OK ' 'to verify your game cache. You can then ' 'export again.').format(file=file + ext, ), master=TK_ROOT, ): webbrowser.open('steam://validate/' + str(self.steamID)) return False, vpk_success if should_backup: LOGGER.info('Backing up original {}!', name) shutil.copy(item_path, backup_path) export_screen.step('BACK') # Backup puzzles, if desired backup.auto_backup(selected_game, export_screen) # Special-case: implement the UnlockDefault stlylevar here, # so all items are modified. if selected_objects['StyleVar']['UnlockDefault']: LOGGER.info('Unlocking Items!') for i, item in enumerate(all_items): # If the Unlock Default Items stylevar is enabled, we # want to force the corridors and obs room to be # deletable and copyable # Also add DESIRES_UP, so they place in the correct orientation if item.id in _UNLOCK_ITEMS: all_items[i] = copy.copy(item) item.deletable = item.copiable = True item.facing = editoritems.DesiredFacing.UP LOGGER.info('Editing Gameinfo...') self.edit_gameinfo(True) export_screen.step('EXP') if not GEN_OPTS.get_bool('General', 'preserve_bee2_resource_dir'): LOGGER.info('Adding ents to FGD.') self.edit_fgd(True) export_screen.step('EXP') # AtomicWriter writes to a temporary file, then renames in one step. # This ensures editoritems won't be half-written. LOGGER.info('Writing Editoritems script...') with srctools.AtomicWriter( self.abs_path('portal2_dlc2/scripts/editoritems.txt') ) as editor_file: editoritems.Item.export(editor_file, all_items, renderables) export_screen.step('EXP') LOGGER.info('Writing Editoritems database...') with open(self.abs_path('bin/bee2/editor.bin'), 'wb') as inst_file: pick = pickletools.optimize(pickle.dumps(all_items)) inst_file.write(pick) export_screen.step('EXP') LOGGER.info('Writing VBSP Config!') os.makedirs(self.abs_path('bin/bee2/'), exist_ok=True) with open(self.abs_path('bin/bee2/vbsp_config.cfg'), 'w', encoding='utf8') as vbsp_file: for line in vbsp_config.export(): vbsp_file.write(line) export_screen.step('EXP') if num_compiler_files > 0: LOGGER.info('Copying Custom Compiler!') compiler_src = utils.install_path('compiler') for comp_file in compiler_src.rglob('*'): # Ignore folders. if comp_file.is_dir(): continue dest = self.abs_path('bin' / comp_file.relative_to(compiler_src)) LOGGER.info('\t* {} -> {}', comp_file, dest) folder = Path(dest).parent if not folder.exists(): folder.mkdir(parents=True, exist_ok=True) try: if os.path.isfile(dest): # First try and give ourselves write-permission, # if it's set read-only. utils.unset_readonly(dest) shutil.copy(comp_file, dest) except PermissionError: # We might not have permissions, if the compiler is currently # running. export_screen.reset() messagebox.showerror( title=_('BEE2 - Export Failed!'), message=_('Copying compiler file {file} failed. ' 'Ensure {game} is not running.').format( file=comp_file, game=self.name, ), master=TK_ROOT, ) return False, vpk_success export_screen.step('COMP') if should_refresh: LOGGER.info('Copying Resources!') music_files = self.copy_mod_music() self.refresh_cache(music_files) LOGGER.info('Optimizing editor models...') self.clean_editor_models(all_items) export_screen.step('EXP') self.generate_fizzler_sides(vbsp_config) if self.steamID == utils.STEAM_IDS['APERTURE TAG']: os.makedirs(self.abs_path('sdk_content/maps/instances/bee2/'), exist_ok=True) with open( self.abs_path( 'sdk_content/maps/instances/bee2/tag_coop_gun.vmf' ), 'w') as f: TAG_COOP_INST_VMF.export(f) export_screen.reset() # Hide loading screen, we're done return True, vpk_success except loadScreen.Cancelled: return False, False