def save(self, ignore_readonly=False):
"""Save the palette file into the specified location.
If ignore_readonly is true, this will ignore the `prevent_overwrite`
property of the palette (allowing resaving those properties over old
versions). Otherwise those palettes always create a new file.
"""
LOGGER.info('Saving "{}"!', self.name)
props = Property(None, [
Property('Name', self.name),
Property('TransName', self.trans_name),
Property('ReadOnly', srctools.bool_as_int(self.prevent_overwrite)),
Property('Items', [
Property(item_id, str(subitem))
for item_id, subitem in self.pos
])
])
# If default, don't include in the palette file.
# Remove the translated name, in case it's not going to write
# properly to the file.
if self.trans_name:
props['Name'] = ''
else:
del props['TransName']
if not self.prevent_overwrite:
del props['ReadOnly']
if self.settings is not None:
self.settings.name = 'Settings'
props.append(self.settings.copy())
# We need to write a new file, determine a valid path.
# Use a hash to ensure it's a valid path (without '-' if negative)
# If a conflict occurs, add ' ' and hash again to get a different
# value.
if self.filename is None or (self.prevent_overwrite
and not ignore_readonly):
hash_src = self.name
while True:
hash_filename = str(abs(hash(hash_src))) + PAL_EXT
if os.path.isfile(hash_filename):
# Add a random character to iterate the hash.
hash_src += chr(random.randrange(0x10ffff))
else:
file = open(os.path.join(PAL_DIR, hash_filename),
'w',
encoding='utf8')
self.filename = os.path.join(PAL_DIR, hash_filename)
break
else:
file = open(os.path.join(PAL_DIR, self.filename),
'w',
encoding='utf8')
with file:
for line in props.export():
file.write(line)
def save(self, ignore_readonly=False):
"""Save the palette file into the specified location.
If ignore_readonly is true, this will ignore the `prevent_overwrite`
property of the palette (allowing resaving those properties over old
versions). Otherwise those palettes always create a new file.
"""
LOGGER.info('Saving "{}"!', self.name)
props = Property(None, [
Property('Name', self.name),
Property('TransName', self.trans_name),
Property('ReadOnly', srctools.bool_as_int(self.prevent_overwrite)),
Property('Items', [
Property(item_id, str(subitem))
for item_id, subitem in self.pos
])
])
# If default, don't include in the palette file.
# Remove the translated name, in case it's not going to write
# properly to the file.
if self.trans_name:
props['Name'] = ''
else:
del props['TransName']
if not self.prevent_overwrite:
del props['ReadOnly']
if self.settings is not None:
self.settings.name = 'Settings'
props.append(self.settings.copy())
# We need to write a new file, determine a valid path.
# Use a hash to ensure it's a valid path (without '-' if negative)
# If a conflict occurs, add ' ' and hash again to get a different
# value.
if self.filename is None or (self.prevent_overwrite and not ignore_readonly):
hash_src = self.name
while True:
hash_filename = str(abs(hash(hash_src))) + PAL_EXT
if os.path.isfile(hash_filename):
# Add a random character to iterate the hash.
hash_src += chr(random.randrange(0x10ffff))
else:
file = open(os.path.join(PAL_DIR, hash_filename), 'w', encoding='utf8')
self.filename = os.path.join(PAL_DIR, hash_filename)
break
else:
file = open(os.path.join(PAL_DIR, self.filename), 'w', encoding='utf8')
with file:
for line in props.export():
file.write(line)
def gen_sound_manifest(additional, excludes):
"""Generate a new game_sounds_manifest.txt file.
This includes all the current scripts defined, plus any custom ones.
Excludes is a list of scripts to remove from the listing - this allows
overriding the sounds without VPK overrides.
"""
if not additional:
return # Don't pack, there aren't any new sounds..
orig_manifest = os.path.join(
'..',
SOUND_MAN_FOLDER.get(CONF['game_id', ''], 'portal2'),
'scripts',
'game_sounds_manifest.txt',
)
try:
with open(orig_manifest) as f:
props = Property.parse(f, orig_manifest).find_key(
'game_sounds_manifest',
[],
)
except FileNotFoundError: # Assume no sounds
props = Property('game_sounds_manifest', [])
scripts = [prop.value for prop in props.find_all('precache_file')]
for script in additional:
scripts.append(script)
for script in excludes:
try:
scripts.remove(script)
except ValueError:
LOGGER.warning(
'"{}" should be excluded, but it\'s'
' not in the manifest already!',
script,
)
# Build and unbuild it to strip other things out - Valve includes a bogus
# 'new_sound_scripts_must_go_below_here' entry..
new_props = Property('game_sounds_manifest',
[Property('precache_file', file) for file in scripts])
inject_loc = os.path.join('bee2', 'inject', 'soundscript_manifest.txt')
with open(inject_loc, 'w') as f:
for line in new_props.export():
f.write(line)
LOGGER.info('Written new soundscripts_manifest..')
def gen_part_manifest(additional):
"""Generate a new particle system manifest file.
This includes all the current ones defined, plus any custom ones.
"""
if not additional:
return # Don't pack, there aren't any new particles..
orig_manifest = os.path.join(
'..',
GAME_FOLDER.get(CONF['game_id', ''], 'portal2'),
'particles',
'particles_manifest.txt',
)
try:
with open(orig_manifest) as f:
props = Property.parse(f, orig_manifest).find_key(
'particles_manifest',
[],
)
except FileNotFoundError: # Assume no particles
props = Property('particles_manifest', [])
parts = [prop.value for prop in props.find_all('file')]
for particle in additional:
parts.append(particle)
# Build and unbuild it to strip comments and similar lines.
new_props = Property('particles_manifest',
[Property('file', file) for file in parts])
inject_loc = os.path.join('bee2', 'inject', 'particles_manifest.txt')
with open(inject_loc, 'w') as f:
for line in new_props.export():
f.write(line)
LOGGER.info('Written new particles_manifest..')
def export(exp_data: ExportData) -> None:
"""Export all the packlists."""
pack_block = Property('PackList', [])
for pack in PackList.all(): # type: PackList
# Build a
# "Pack_id"
# {
# "File" "filename"
# "File" "filename"
# }
# block for each packlist
files = [Property('File', file) for file in pack.files]
pack_block.append(Property(
pack.id,
files,
))
LOGGER.info('Writing packing list!')
with open(exp_data.game.abs_path('bin/bee2/pack_list.cfg'),
'w') as pack_file:
for line in pack_block.export():
pack_file.write(line)
def build_instance_data(editoritems: Property):
"""Build a property tree listing all of the instances for each item.
as well as another listing the input and output commands.
VBSP uses this to reduce duplication in VBSP_config files.
This additionally strips custom instance definitions from the original
list.
"""
instance_locs = Property("AllInstances", [])
cust_inst = Property("CustInstances", [])
commands = Property("Connections", [])
item_classes = Property("ItemClasses", [])
root_block = Property(None, [
instance_locs,
item_classes,
cust_inst,
commands,
])
for item in editoritems.find_all("Item"):
instance_block = Property(item['Type'], [])
instance_locs.append(instance_block)
comm_block = Property(item['Type'], [])
for inst_block in item.find_all("Exporting", "instances"):
for inst in inst_block.value[:]: # type: Property
if inst.name.isdigit():
# Direct Portal 2 value
instance_block.append(
Property('Instance', inst['Name'])
)
else:
# It's a custom definition, remove from editoritems
inst_block.value.remove(inst)
# Allow the name to start with 'bee2_' also to match
# the <> definitions - it's ignored though.
name = inst.name
if name[:5] == 'bee2_':
name = name[5:]
cust_inst.set_key(
(item['type'], name),
# Allow using either the normal block format,
# or just providing the file - we don't use the
# other values.
inst['name'] if inst.has_children() else inst.value,
)
# Look in the Inputs and Outputs blocks to find the io definitions.
# Copy them to property names like 'Input_Activate'.
for io_type in ('Inputs', 'Outputs'):
for block in item.find_all('Exporting', io_type, CONN_NORM):
for io_prop in block:
comm_block[
io_type[:-1] + '_' + io_prop.real_name
] = io_prop.value
# The funnel item type is special, having the additional input type.
# Handle that specially.
if item['type'].casefold() == 'item_tbeam':
for block in item.find_all('Exporting', 'Inputs', CONN_FUNNEL):
for io_prop in block:
comm_block['TBeam_' + io_prop.real_name] = io_prop.value
# Fizzlers don't work correctly with outputs. This is a signal to
# conditions.fizzler, but it must be removed in editoritems.
if item['ItemClass', ''].casefold() == 'itembarrierhazard':
for block in item.find_all('Exporting', 'Outputs'):
if CONN_NORM in block:
del block[CONN_NORM]
# Record the itemClass for each item type.
item_classes[item['type']] = item['ItemClass', 'ItemBase']
# Only add the block if the item actually has IO.
if comm_block.value:
commands.append(comm_block)
return root_block.export()
def decompile_model(
fsys: FileSystemChain,
cache_loc: Path,
crowbar: Path,
filename: str,
checksum: bytes,
) -> Optional[QC]:
"""Use Crowbar to decompile models directly for propcombining."""
cache_folder = cache_loc / Path(filename).with_suffix('')
info_path = cache_folder / 'info.kv'
if cache_folder.exists():
try:
with info_path.open() as f:
cache_props = Property.parse(f).find_key('qc', [])
except (FileNotFoundError, KeyValError):
pass
else:
# Previous compilation.
if checksum == bytes.fromhex(cache_props['checksum', '']):
ref_smd = cache_props['ref', '']
if not ref_smd:
return None
phy_smd = cache_props['phy', None]
if phy_smd is not None:
phy_smd = str(cache_folder / phy_smd)
return QC(
str(info_path),
str(cache_folder / ref_smd),
phy_smd,
cache_props.float('ref_scale', 1.0),
cache_props.float('phy_scale', 1.0),
)
# Otherwise, re-decompile.
LOGGER.info('Decompiling {}...', filename)
qc: Optional[QC] = None
# Extract out the model to a temp dir.
with TemporaryDirectory() as tempdir, fsys:
stem = Path(filename).stem
filename_no_ext = filename[:-4]
for mdl_ext in MDL_EXTS:
try:
file = fsys[filename_no_ext + mdl_ext]
except FileNotFoundError:
pass
else:
with file.open_bin() as src, Path(tempdir, stem +
mdl_ext).open('wb') as dest:
shutil.copyfileobj(src, dest)
LOGGER.debug('Extracted "{}" to "{}"', filename, tempdir)
args = [
str(crowbar),
'decompile',
'-i',
str(Path(tempdir, stem + '.mdl')),
'-o',
str(cache_folder),
]
LOGGER.debug('Executing {}', ' '.join(args))
result = subprocess.run(args)
if result.returncode != 0:
LOGGER.warning('Could not decompile "{}"!', filename)
return None
# There should now be a QC file here.
for qc_path in cache_folder.glob('*.qc'):
qc_result = parse_qc(cache_folder, qc_path)
break
else: # not found.
LOGGER.warning('No QC outputted into {}', cache_folder)
qc_result = None
qc_path = Path()
cache_props = Property('qc', [])
cache_props['checksum'] = checksum.hex()
if qc_result is not None:
(
model_name,
ref_scale,
ref_smd,
phy_scale,
phy_smd,
) = qc_result
qc = QC(
str(qc_path).replace('\\', '/'),
str(ref_smd).replace('\\', '/'),
str(phy_smd).replace('\\', '/') if phy_smd else None,
ref_scale,
phy_scale,
)
cache_props['ref'] = Path(ref_smd).name
cache_props['ref_scale'] = format(ref_scale, '.6g')
if phy_smd is not None:
cache_props['phy'] = Path(phy_smd).name
cache_props['phy_scale'] = format(phy_scale, '.6g')
else:
cache_props['ref'] = '' # Mark as not present.
with info_path.open('w') as f:
for line in cache_props.export():
f.write(line)
return qc
def build_instance_data(editoritems: Property):
"""Build a property tree listing all of the instances for each item.
as well as another listing the input and output commands.
VBSP uses this to reduce duplication in VBSP_config files.
This additionally strips custom instance definitions from the original
list.
"""
instance_locs = Property("AllInstances", [])
cust_inst = Property("CustInstances", [])
commands = Property("Connections", [])
item_classes = Property("ItemClasses", [])
root_block = Property(None, [instance_locs, item_classes, cust_inst, commands])
for item in editoritems.find_all("Item"):
instance_block = Property(item["Type"], [])
instance_locs.append(instance_block)
comm_block = Property(item["Type"], [])
for inst_block in item.find_all("Exporting", "instances"):
for inst in inst_block.value[:]: # type: Property
if inst.name.isdigit():
# Direct Portal 2 value
instance_block.append(Property("Instance", inst["Name"]))
else:
# It's a custom definition, remove from editoritems
inst_block.value.remove(inst)
# Allow the name to start with 'bee2_' also to match
# the <> definitions - it's ignored though.
name = inst.name
if name[:5] == "bee2_":
name = name[5:]
cust_inst.set_key(
(item["type"], name),
# Allow using either the normal block format,
# or just providing the file - we don't use the
# other values.
inst["name"] if inst.has_children() else inst.value,
)
# Look in the Inputs and Outputs blocks to find the io definitions.
# Copy them to property names like 'Input_Activate'.
for io_type in ("Inputs", "Outputs"):
for block in item.find_all("Exporting", io_type, CONN_NORM):
for io_prop in block:
comm_block[io_type[:-1] + "_" + io_prop.real_name] = io_prop.value
# The funnel item type is special, having the additional input type.
# Handle that specially.
if item["type"] == "item_tbeam":
for block in item.find_all("Exporting", "Inputs", CONN_FUNNEL):
for io_prop in block:
comm_block["TBEAM_" + io_prop.real_name] = io_prop.value
# Fizzlers don't work correctly with outputs. This is a signal to
# conditions.fizzler, but it must be removed in editoritems.
if item["ItemClass", ""].casefold() == "itembarrierhazard":
for block in item.find_all("Exporting", "Outputs"):
if CONN_NORM in block:
del block[CONN_NORM]
# Record the itemClass for each item type.
item_classes[item["type"]] = item["ItemClass", "ItemBase"]
# Only add the block if the item actually has IO.
if comm_block.value:
commands.append(comm_block)
return root_block.export()
def export(
self,
style: packages.Style,
selected_objects: dict,
should_refresh=False,
) -> Tuple[bool, bool]:
"""Generate the editoritems.txt and vbsp_config.
- If no backup is present, the original editoritems is backed up.
- For each object type, run its .export() function with the given
- item.
- Styles are a special case.
"""
LOGGER.info('-' * 20)
LOGGER.info('Exporting Items and Style for "{}"!', self.name)
LOGGER.info('Style = {}', style.id)
for obj, selected in selected_objects.items():
# Skip the massive dict in items
if obj == 'Item':
selected = selected[0]
LOGGER.info('{} = {}', obj, selected)
# VBSP, VRAD, editoritems
export_screen.set_length('BACK', len(FILES_TO_BACKUP))
# files in compiler/
try:
num_compiler_files = sum(
1 for file in utils.install_path('compiler').rglob('*'))
except FileNotFoundError:
num_compiler_files = 0
if self.steamID == utils.STEAM_IDS['APERTURE TAG']:
# Coop paint gun instance
num_compiler_files += 1
if num_compiler_files == 0:
LOGGER.warning('No compiler files!')
export_screen.skip_stage('COMP')
else:
export_screen.set_length('COMP', num_compiler_files)
LOGGER.info('Should refresh: {}', should_refresh)
if should_refresh:
# Check to ensure the cache needs to be copied over..
should_refresh = self.cache_invalid()
if should_refresh:
LOGGER.info("Cache invalid - copying..")
else:
LOGGER.info("Skipped copying cache!")
# Each object type
# Editoritems
# VBSP_config
# Instance list
# Editor models.
# FGD file
# Gameinfo
export_screen.set_length('EXP', len(packages.OBJ_TYPES) + 6)
# Do this before setting music and resources,
# those can take time to compute.
export_screen.show()
try:
if should_refresh:
# Count the files.
export_screen.set_length(
'RES',
sum(1 for file in res_system.walk_folder_repeat()),
)
else:
export_screen.skip_stage('RES')
export_screen.skip_stage('MUS')
# Make the folders we need to copy files to, if desired.
os.makedirs(self.abs_path('bin/bee2/'), exist_ok=True)
# Start off with the style's data.
vbsp_config = Property(None, [])
vbsp_config += style.config.copy()
all_items = style.items.copy()
renderables = style.renderables.copy()
export_screen.step('EXP')
vpk_success = True
# Export each object type.
for obj_name, obj_data in packages.OBJ_TYPES.items():
if obj_name == 'Style':
continue # Done above already
LOGGER.info('Exporting "{}"', obj_name)
selected = selected_objects.get(obj_name, None)
try:
obj_data.cls.export(
packages.ExportData(
game=self,
selected=selected,
all_items=all_items,
renderables=renderables,
vbsp_conf=vbsp_config,
selected_style=style,
))
except packages.NoVPKExport:
# Raised by StyleVPK to indicate it failed to copy.
vpk_success = False
export_screen.step('EXP')
vbsp_config.set_key(('Options', 'Game_ID'), self.steamID)
vbsp_config.set_key(
('Options', 'dev_mode'),
srctools.bool_as_int(optionWindow.DEV_MODE.get()))
# If there are multiple of these blocks, merge them together.
# They will end up in this order.
vbsp_config.merge_children(
'Textures',
'Fizzlers',
'Options',
'StyleVars',
'DropperItems',
'Conditions',
'Quotes',
'PackTriggers',
)
for name, file, ext in FILES_TO_BACKUP:
item_path = self.abs_path(file + ext)
backup_path = self.abs_path(file + '_original' + ext)
if not os.path.isfile(item_path):
# We can't backup at all.
should_backup = False
elif name == 'Editoritems':
should_backup = not os.path.isfile(backup_path)
else:
# Always backup the non-_original file, it'd be newer.
# But only if it's Valves - not our own.
should_backup = should_backup_app(item_path)
backup_is_good = should_backup_app(backup_path)
LOGGER.info(
'{}{}: normal={}, backup={}',
file,
ext,
'Valve' if should_backup else 'BEE2',
'Valve' if backup_is_good else 'BEE2',
)
if not should_backup and not backup_is_good:
# It's a BEE2 application, we have a problem.
# Both the real and backup are bad, we need to get a
# new one.
try:
os.remove(backup_path)
except FileNotFoundError:
pass
try:
os.remove(item_path)
except FileNotFoundError:
pass
export_screen.reset()
if messagebox.askokcancel(
title=_('BEE2 - Export Failed!'),
message=_(
'Compiler file {file} missing. '
'Exit Steam applications, then press OK '
'to verify your game cache. You can then '
'export again.').format(file=file + ext, ),
master=TK_ROOT,
):
webbrowser.open('steam://validate/' +
str(self.steamID))
return False, vpk_success
if should_backup:
LOGGER.info('Backing up original {}!', name)
shutil.copy(item_path, backup_path)
export_screen.step('BACK')
# Backup puzzles, if desired
backup.auto_backup(selected_game, export_screen)
# Special-case: implement the UnlockDefault stlylevar here,
# so all items are modified.
if selected_objects['StyleVar']['UnlockDefault']:
LOGGER.info('Unlocking Items!')
for i, item in enumerate(all_items):
# If the Unlock Default Items stylevar is enabled, we
# want to force the corridors and obs room to be
# deletable and copyable
# Also add DESIRES_UP, so they place in the correct orientation
if item.id in _UNLOCK_ITEMS:
all_items[i] = copy.copy(item)
item.deletable = item.copiable = True
item.facing = editoritems.DesiredFacing.UP
LOGGER.info('Editing Gameinfo...')
self.edit_gameinfo(True)
export_screen.step('EXP')
if not GEN_OPTS.get_bool('General', 'preserve_bee2_resource_dir'):
LOGGER.info('Adding ents to FGD.')
self.edit_fgd(True)
export_screen.step('EXP')
# AtomicWriter writes to a temporary file, then renames in one step.
# This ensures editoritems won't be half-written.
LOGGER.info('Writing Editoritems script...')
with srctools.AtomicWriter(
self.abs_path('portal2_dlc2/scripts/editoritems.txt')
) as editor_file:
editoritems.Item.export(editor_file, all_items, renderables)
export_screen.step('EXP')
LOGGER.info('Writing Editoritems database...')
with open(self.abs_path('bin/bee2/editor.bin'), 'wb') as inst_file:
pick = pickletools.optimize(pickle.dumps(all_items))
inst_file.write(pick)
export_screen.step('EXP')
LOGGER.info('Writing VBSP Config!')
os.makedirs(self.abs_path('bin/bee2/'), exist_ok=True)
with open(self.abs_path('bin/bee2/vbsp_config.cfg'),
'w',
encoding='utf8') as vbsp_file:
for line in vbsp_config.export():
vbsp_file.write(line)
export_screen.step('EXP')
if num_compiler_files > 0:
LOGGER.info('Copying Custom Compiler!')
compiler_src = utils.install_path('compiler')
for comp_file in compiler_src.rglob('*'):
# Ignore folders.
if comp_file.is_dir():
continue
dest = self.abs_path('bin' /
comp_file.relative_to(compiler_src))
LOGGER.info('\t* {} -> {}', comp_file, dest)
folder = Path(dest).parent
if not folder.exists():
folder.mkdir(parents=True, exist_ok=True)
try:
if os.path.isfile(dest):
# First try and give ourselves write-permission,
# if it's set read-only.
utils.unset_readonly(dest)
shutil.copy(comp_file, dest)
except PermissionError:
# We might not have permissions, if the compiler is currently
# running.
export_screen.reset()
messagebox.showerror(
title=_('BEE2 - Export Failed!'),
message=_('Copying compiler file {file} failed. '
'Ensure {game} is not running.').format(
file=comp_file,
game=self.name,
),
master=TK_ROOT,
)
return False, vpk_success
export_screen.step('COMP')
if should_refresh:
LOGGER.info('Copying Resources!')
music_files = self.copy_mod_music()
self.refresh_cache(music_files)
LOGGER.info('Optimizing editor models...')
self.clean_editor_models(all_items)
export_screen.step('EXP')
self.generate_fizzler_sides(vbsp_config)
if self.steamID == utils.STEAM_IDS['APERTURE TAG']:
os.makedirs(self.abs_path('sdk_content/maps/instances/bee2/'),
exist_ok=True)
with open(
self.abs_path(
'sdk_content/maps/instances/bee2/tag_coop_gun.vmf'
), 'w') as f:
TAG_COOP_INST_VMF.export(f)
export_screen.reset() # Hide loading screen, we're done
return True, vpk_success
except loadScreen.Cancelled:
return False, False
