def _Build_Engine_Objects():
'''
Returns a list of Edit_Objects for 'assets/props/Engines'.
Meant for calling from the Live_Editor.
'''
File_System.Load_Files('*assets/props/Engines/*.xml')
game_files = File_System.Get_Asset_Files_By_Class('macros', 'engine')
return Create_Objects_From_Asset_Files(game_files, engine_item_macros)
def _Build_Storage_Objects():
# These are a bit scattered, some in the units folder and
# some in storagemodules. Use a name pattern match for this.
File_System.Get_All_Indexed_Files('macros', 'storage_*')
#File_System.Load_Files('*assets/props/StorageModules/*.xml')
# Followup with class check for safety.
game_files = File_System.Get_Asset_Files_By_Class('macros', 'storage')
return Create_Objects_From_Asset_Files(game_files, storage_item_macros)
def Action_Reset(self):
'''
The Reset button was pressed.
This will reset the file system, live editor, and refresh all tabs.
'''
# Verify no threads are active; if they are, ignore the button press.
# This should normally be impossible if the action was disabled
# properly.
if self.worker_thread.thread_active:
return
# Kick off a general save for all tabs.
# This saves settings, enabled extensions, maybe some others.
# After reset some tabs will reload from disk, so this ensures
# those changes are preserved.
self.Send_Signal('save')
# Reset the file system to flush out old contents.
File_System.Reset()
# Reset the live editor, to make it do a fresh pull of data
# from the file system.
# Note: this is designed to remember its patches across a reset,
# so they don't need to be saved.
Live_Editor.Reset()
# Signal all tabs to reset, but only if the settings paths
# are set up properly.
if Settings.Paths_Are_Valid():
self.Send_Signal('file_system_reset')
return
def Get_Macros(self, pattern, classes=None, class_names=None):
'''
Returns a list of Macros with names matching the given pattern.
* pattern
- Wildcard virtual_path matching pattern.
* classes
- Optional list of macro classes to include.
* class_names
- Optional list of class names to include.
'''
# Cache patterns seen, to skip index lookup.
if pattern not in self._get_macros_cache:
self._get_macros_cache.add(pattern)
# The index will match macro names to files, where a file can
# hold multiple macros. Start by getting the files.
game_files = File_System.Get_All_Indexed_Files('macros', pattern)
# Add each file to the database, loading macros, xml, etc, skipping
# those already loaded.
for game_file in game_files:
self.Load_File(game_file)
# Now pick out the actual macros.
macro_names = fnmatch.filter(self.macros.keys(), pattern.lower())
# Filter for wanted classes, if a list was given.
return [
self.macros[x] for x in macro_names
if ((not class_names or self.macros[x].class_name in class_names)
and (not classes or isinstance(self.macros[x], tuple(classes)))
)
]
def Display_Update_Factory_Name(t_factory):
'Look up factory name.'
if t_factory:
#t_file = File_System.Load_File('t/0001-L044.xml')
# Let the t-file Read handle the lookup.
#return t_file.Read(t_factory)
return File_System.Read_Text(t_factory)
return
def Update_Description(t_descrip_entry, ):
'Look up a text reference description.'
if not t_descrip_entry:
return ''
else:
#t_file = File_System.Load_File('t/0001-L044.xml')
# Let the t-file Read handle the lookup.
#return t_file.Read(t_descrip_entry)
return File_System.Read_Text(t_descrip_entry)
def Test_Thread(self, extension_name_list):
'''
Threaded tester. This will block access by other tabs to
the file system while running.
Emits 'test_result' signals as extension tests complete.
'''
# Reset the file system completely.
# TODO: do this only if the extensions enabled are changed.
# TODO: can alternately set up the file system to track all
# extensions, locally skipping those disabled or ignored,
# and just change that behavior during test loads, such that
# prior state is preserved (though a reset may be needed if
# actual enabled extensions are changed).
File_System.Reset()
# Temporary overrides of Settings so that all enabled
# extensions are loaded.
# Include the current output extension in the check.
old_ignore_extensions = Settings.ignore_extensions
old_ignore_output_extension = Settings.ignore_output_extension
Settings.ignore_extensions = False
Settings.ignore_output_extension = False
# Run the tests, collecting the logged messages.
#ext_log_lines_dict = {}
for extension_name in extension_name_list:
log_lines = Check_Extension(extension_name,
return_log_messages=True)
# Make the gui more responsive during testing by
# using a signal emitted to a function that catches results
# and updates state as each extension finishes.
self.test_result.emit(extension_name, '\n'.join(log_lines))
# Restore the Settings.
Settings.ignore_extensions = old_ignore_extensions
Settings.ignore_output_extension = old_ignore_output_extension
# Reset the file system again, so it can restore old extension
# finding logic.
File_System.Reset()
return #ext_log_lines_dict
def Get_Ship_Macro_Files():
'''
Finds and returns all ship macro game files.
'''
# Note: normally ship macros start with "ship_".
# However, this doesn't find the XR shippack, since they all start
# with "units_" instead; also over-includes on XR shippack since it calls
# some docking bay macros "ship_storage_".
File_System.Get_All_Indexed_Files('macros', 'ship_*')
File_System.Get_All_Indexed_Files('macros', 'units_*')
# Search out loaded assets, which already have checked the
# class tags. Do this for each ship class.
ret_list = []
for suffix in ['xs', 's', 'm', 'l', 'xl']:
ret_list += File_System.Get_Asset_Files_By_Class(
'macros', f'ship_{suffix}')
return ret_list
def Color_Text(*page_t_colors):
'''
Applies coloring to selected text nodes, for all versions
of the text found in the current X4 files.
Note: these colors will override any prior color in effect,
and will return to standard text color at the end of the colored
text node.
* page_t_colors
- One or more groups of (page id, text id, color code) to apply.
Example:
<code>
Color_Text(
(20005,1001,'B'),
(20005,3012,'C'),
(20005,6046,'Y'),
)
</code>
'''
# TODO: add a list of support colors to the doc.
# TODO: verify input.
# Load all text files.
game_files = File_System.Load_Files('t/*.xml')
# Loop over them.
for game_file in game_files:
xml_root = game_file.Get_Root()
# Loop over the colorings.
change_found = False
for page, text, color in page_t_colors:
# Look up the node.
node = xml_root.find('./page[@id="{}"]/t[@id="{}"]'.format(
page, text))
# Skip on missing node.
if node == None:
continue
# Prefix and suffix it with color.
node.text = r'\033{}{}\033X'.format(color, node.text)
change_found = True
# TODO: delay this until after all loops complete, in
# case a later one has an error, to safely cancel the
# whole transform.
if change_found:
game_file.Update_Root(xml_root)
return
# TODO: develop this further to support text searching, coloring only
# selected words, maybe; that wouldn't be as robust across languages
# though.
def Display_Update_Ware_Name(t_name_entry, id):
'Look up ware name.'
# If no t_name_entry available, use the id name.
if not t_name_entry:
return id
else:
#t_file = File_System.Load_File('t/0001-L044.xml')
# Let the t-file Read handle the lookup.
#name = t_file.Read(t_name_entry)
name = File_System.Read_Text(t_name_entry)
return name
def Get_Macro_Component_File(macro_file, xpath_prefix):
'''
Returns the component file matched to the given macro file,
along with an xpath to the component node.
The component should already have been loaded.
TODO: maybe support an attempt at autoloading one directory up.
'''
root = macro_file.Get_Root_Readonly()
component_name = root.xpath(xpath_prefix + '/component')[0].get('ref')
component_file = File_System.Get_Indexed_File('components', component_name)
xpath = component_file.Get_Asset_Xpath(component_name)
return component_file, xpath
def _Build_Weapon_Objects():
'''
Returns a list of Edit_Objects for all found weapons.
Meant for calling from the Live_Editor.
'''
# Make sure the bullets are created, so they can be referenced.
Live_Editor.Get_Category_Objects('bullets')
game_files = File_System.Get_Asset_Files_By_Class('macros', 'weapon',
'missilelauncher',
'turret',
'missileturret',
'bomblauncher')
return Create_Objects_From_Asset_Files(game_files, weapon_item_macros)
def Write_Modified_Binaries():
'''
Write out any modified binaries. These are placed in the main x4
folder, not in an extension.
'''
# Return early if settings have writeback disabled.
if Settings.disable_cleanup_and_writeback:
Print('Skipping Write_Extension; writeback disabled in Settings.')
return
# Trigger writeback.
# Don't clear old stuff; this has no associated log, and just overwrites.
File_System.Write_Non_Ext_Files()
return
def Script_Prelaunch(self):
'''
Code to run just prior to a script being launched.
Resets the file system, saves live editor patches, etc.
'''
# Clear out the file system from any prior run changes.
File_System.Reset()
# Save the Live_Editor patches, since they may get loaded
# by a plugin for xml application.
Live_Editor.Save_Patches()
self.window.Print('Saved Live Editor patches')
# Send a signal so Config is updated properly.
self.window.Send_Signal('script_starting')
return
def _Build_Bullet_Objects():
'''
Returns a list of Edit_Objects for all found bullets.
Meant for calling from the Live_Editor.
'''
# Make sure engines are loaded for the missiles.
Live_Editor.Get_Category_Objects('engines')
# Look up bullet files.
# These can be in two locations.
File_System.Load_Files('*assets/props/WeaponSystems/*.xml')
File_System.Load_Files('*assets/fx/weaponFx/*.xml')
# Split out proper bullets from missiles and similar.
bullet_game_files = File_System.Get_Asset_Files_By_Class(
'macros', 'bullet')
missile_game_files = File_System.Get_Asset_Files_By_Class(
'macros', 'missile', 'bomb', 'mine', 'countermeasure')
objects = []
objects += Create_Objects_From_Asset_Files(bullet_game_files,
bullet_item_macros)
objects += Create_Objects_From_Asset_Files(missile_game_files,
missile_item_macros)
return objects
def Write_To_Extension(skip_content = False):
'''
Write all currently modified game files to the extension
folder. Existing files at the location written on a prior
call will be cleared out. Content.xml will have dependencies
added for files modified from existing extensions.
* skip_content
- Bool, if True then the content.xml file will not be written.
- Content is automatically skipped if Make_Extension_Content_XML
was already called.
- Defaults to False.
'''
# TODO: maybe make path and extension name into arguments,
# though for now they should be set up in Settings since
# this is the location log files were written to.
## If no name given, use the one from settings.
#if not extension_name:
# extension_name = Settings.extension_name
# Return early if settings have writeback disabled.
if Settings.disable_cleanup_and_writeback:
Print('Skipping Write_Extension; writeback disabled in Settings.')
return
# Clean old files, based on whatever old log is there.
File_System.Cleanup()
# Create a content.xml game file.
if not skip_content:
Make_Extension_Content_XML()
# Trigger writeback.
File_System.Write_Files()
return
def Get_Game_Name(self):
'''
Return the in-game name of this macro, defaulting to the macro
name if not found.
'''
if not hasattr(self, '_game_name'):
# Default to macro name.
self._game_name = self.name
# Check for idenfication/name.
ident_node = self.xml_node.find('./properties/identification')
if ident_node != None:
name_code = ident_node.get('name')
if name_code:
self._game_name = File_System.Read_Text(name_code)
return self._game_name
def Update_Name(t_name_entry, codename):
'Look up a text reference name.'
if not t_name_entry:
# If no t_name_entry available, use the codename name.
# Component could be used sometimes, since it often is the codename
# without a _macro suffix, but doesn't work quite as well
# since sometimes objects will share a component (causing them
# to have the same name).
# This will manually prune the _macro term if found.
return codename.replace('_macro', '')
else:
#t_file = File_System.Load_File('t/0001-L044.xml')
#name = t_file.Read(t_name_entry)
# Let the t-file Read handle the lookup.
name = File_System.Read_Text(t_name_entry)
# TODO: maybe replace with something when the name is blank.
return name
def _Get_Ware_Node(self):
'''
Support function that returns a wares.xml node associated with
this macro, or None. If there are multiple matches, returns the first.
'''
# TODO: add ware nodes to Database, and link to there.
if not hasattr(self, '_ware_node'):
wares_file = File_System.Load_File('libraries/wares.xml')
xml_root = wares_file.Get_Root_Readonly()
# Stupidly, the "component" node holds this macro name.
ware_entries = xml_root.xpath(
f'./ware[./component/@ref="{self.name}"]')
if ware_entries:
# Assume just one match.
self._ware_node = ware_entries[0]
else:
self._ware_node = None
return self._ware_node
def Get_Components(self, pattern):
'''
Returns a list of Components with names matching the given pattern.
'''
# Cache patterns seen, to skip index lookup.
if pattern not in self._get_components_cache:
self._get_components_cache.add(pattern)
# The index will match names to files, where a file can
# hold multiple macros. Start by getting the files.
game_files = File_System.Get_All_Indexed_Files(
'components', pattern)
# Add each file to the database, loading macros, xml, etc, skipping
# those already loaded.
for game_file in game_files:
self.Load_File(game_file)
# Now pick out the actual components.
component_names = fnmatch.filter(self.components.keys(),
pattern.lower())
return [self.components[x] for x in component_names]
def Update_Content_XML_Dependencies():
'''
Update the dependencies in the content xml file, based on which other
extensions touched files modified by the current script.
If applied to an existing content.xml (not one created here), existing
dependencies are kept, and only customizer dependencies are updated.
Note: an existing xml file may loose custom formatting.
'''
# TODO: framework needs more development to handle cases with an
# existing content.xml cleanly, since currently the output extension is
# always ignored, and there is no particular method of dealing with
# output-ext new files not having an extensions/... path.
# Try to load a locally created content.xml.
content_file = Load_File('content.xml', error_if_not_found = False)
# If not found, then search for an existing content.xml on disk.
if not content_file:
# Manually load it.
content_path = Settings.Get_Output_Folder() / 'content.xml'
# Verify the file exists.
if not content_path.exists():
Print('Error in Update_Content_XML_Dependencies: could not find an existing content.xml file')
return
content_file = File_System.Add_File( XML_File(
# Plain file name as path, since this will write back to the
# extension folder.
virtual_path = 'content.xml',
binary = content_path.read_bytes(),
# Edit the existing file.
edit_in_place = True,
))
root = content_file.Get_Root()
# Set the ID based on replacing spaces.
this_id = Settings.extension_name.replace(' ','_')
# Remove old dependencies from the customizer, and record others.
existing_deps = []
for dep in root.xpath('./dependency'):
if dep.get('from_customizer'):
dep.getparent().remove(dep)
else:
existing_deps.append(dep.get('id'))
# Add in dependencies to existing extensions.
# These should be limited to only those extensions which sourced
# any of the files which were modified.
# TODO: future work can track specific node edits, and set dependencies
# only where transform modified nodes might overlap with extension
# modified nodes.
# Dependencies use extension ids, so this will do name to id
# translation.
# Note: multiple dependencies may share the same ID if those extensions
# have conflicting ids; don't worry about that here.
source_extension_ids = set()
for file_name, game_file in File_System.game_file_dict.items():
if not game_file.modified:
continue
for ext_name in game_file.source_extension_names:
# Translate extension names to ids.
ext_id = File_System.source_reader.extension_source_readers[
ext_name].extension_summary.ext_id
source_extension_ids.add(ext_id)
# Add the elements; keep alphabetical for easy reading.
for ext_id in sorted(source_extension_ids):
# Omit self, just in case; shouldn't come up, but might.
if ext_id == this_id:
Print('Error: output extension appears in its own dependencies,'
' indicating it transformed its own prior output.')
continue
# Skip if already defined.
if ext_id in existing_deps:
continue
# Add it, with a tag to indicate it came from the customizer.
# TODO: optional dependencies?
root.append( ET.Element('dependency', id = ext_id, from_customizer = 'true' ))
content_file.Update_Root(root)
return
def Make_Extension_Content_XML(
name = None,
author = None,
version = None,
save = False,
sync = False,
enabled = True,
description = None,
):
'''
Adds an xml file object defining the content.xml for the top
of the extension. Common fields can be specified, or left as defaults.
* name
- String, display name; defaults to extension folder name.
* author
- String, author name; defaults to X4 Customizer.
* version
- String, version code; defaults to customizer version.
* save
- Bool, True if saves made with the extension require it be enabled.
* sync
- Bool.
* enabled
- Bool, True to default to enabled, False disabled.
* description
- String, extended description to use, for all languages.
- Newlines are automatically converted to " " for display in-game.
'''
# If the content exists already, return early.
if Load_File('content.xml', error_if_not_found = False) != None:
return
if not version:
# Content version needs to have 3+ digits, with the last
# two being sub version. This doesn't mesh will with
# the version in the Change_Log, but can do a simple conversion of
# the top two version numbers.
version_split = Framework.Change_Log.Get_Version().split('.')
# Should have at least 2 version numbers, sometimes a third.
assert len(version_split) >= 2
# This could go awry on weird version strings, but for now just assume
# they are always nice integers, and the second is 1-2 digits.
version_major = version_split[0]
version_minor = version_split[1].zfill(2)
assert len(version_minor) == 2
# Combine together.
version = version_major + version_minor
# If a description given, format slightly.
if description:
# Need to use " " for newlines in-game.
description = description.replace('\n', ' ')
else:
description = ' '
# Set the ID based on replacing spaces.
this_id = Settings.extension_name.replace(' ','_')
# Set up the root content node.
content_node = ET.Element(
'content',
attrib = {
# Swap spaces to _; unclear on if x4 accepts spaces.
'id' : this_id,
'name' : name if name else Settings.extension_name,
'author' : author if author else 'X4_Customizer',
'version' : version,
'date' : File_System.Get_Date(),
# TODO: maybe track when changes are save breaking, eg. if
# adding in new ships or similar. Though in practice, it
# may be best to always keep this false, and offer transforms
# that can undo breaking changes safely before uninstall.
'save' : 'true' if save else 'false',
'sync' : 'true' if sync else 'false',
'enabled' : 'true' if enabled else 'false',
'description': description,
})
# Fill out language description text.
# This loops over all language ids noticed in the cat/dat t files.
for lang_id in ['7','33','34','39','44','48','49','55','81','82','86','88']:
# Set up a new text node.
# TODO: per-language descriptions.
text_node = ET.Element('language', language = lang_id, description = description)
# Add to the content node.
content_node.append(text_node)
# Record it.
File_System.Add_File( XML_File(
xml_root = content_node,
virtual_path = 'content.xml',
modified = True))
# Use shared code to fill in dependencies.
Update_Content_XML_Dependencies()
return
def Remove_Blinking_Ship_Lights():
'''
Removes the blinking lights from ships.
'''
'''
Of interest are the omni nodes that contain animations for blinking.
Example:
' <omni name="XU Omni091" shadow="0" r="255" g="0" b="0" range="2" shadowrange="2" lighteffect="1" trigger="1" intensity="1" specularintensity="1">
' <lightanimations>
' <lightanimation name="intensity" controller="linear_float">
' <key frame="0" value="1"/>
' <key frame="4" value="1"/>
' <key frame="5" value="0"/>
' <key frame="9" value="0"/>
' <key frame="10" value="1"/>
' <key frame="14" value="1"/>
' <key frame="15" value="0"/>
' <key frame="64" value="0"/>
' <key frame="65" value="1"/>
' <key frame="69" value="1"/>
' <key frame="70" value="0"/>
' <key frame="100" value="0"/>
' </lightanimation>
' </lightanimations>
' <offset>
' <position x="-7.476295" y="-0.733321" z="-5.032233"/>
' </offset>
' </omni>
Can do a general search for such nodes.
Note: the blinking rate seems to be similar but not quite the same
between ships, at least the animation key count can differ (12, 13, etc.).
Check for "anim_poslights" in the parent part name, though
this is not consistent across ships (eg. "anim_poslights_left"), but
adds some safety against accidental removal of other components.
Note: just removing the omni nodes has no effect, for whatever reason,
but removing the entire connection node housing them is effective.
(Removing the part also didn't work.)
Update: the split medium miners define the anim_poslights part, but
without accomponying omni lights, and yet still blink.
Assuming the lights are defined elsewhere, removing the anim_poslights
connection should turn off blinking.
TODO: in 3.2 connection names were changed on some ships, which throws
off diff patches which xpath to the connection node by name. Index would
also fail in this case. Both are in danger of matching the wrong
connection, which could cause fundumental problems if stored in a save,
eg. disconnected components (losing shields/whatever).
How can the xpath creator be induced to use a child node/attribute
when removing a parent node?
(Workaround might be to match tags, though no promises on that being safe.)
'''
ship_files = File_System.Get_All_Indexed_Files('components','ship_*')
# Test with just a kestrel
#ship_files = [Load_File('assets/units/size_s/ship_tel_s_scout_01.xml')]
for game_file in ship_files:
xml_root = game_file.Get_Root()
modified = False
# Find the connection that has a uv_animation.
for conn in xml_root.xpath(".//connection[parts/part/uv_animations/uv_animation]"):
# Check the part name for anim_poslights (or a variation).
parts = conn.xpath('./parts/part')
# Expecting just one part.
if len(parts) > 1:
print('Error: multiple parts matched in {}, skipping'.format(game_file.virtual_path))
part_name = parts[0].get('name')
if 'anim_poslights' not in part_name:
continue
# Remove it from its parent.
conn.getparent().remove(conn)
modified = True
if modified:
# Commit changes right away; don't bother delaying for errors.
game_file.Update_Root(xml_root)
# Encourage a better xpath match rule.
game_file.Add_Forced_Xpath_Attributes('parts/part/@name')
return
def Run():
# Test if the gui window is open; error message if so.
from Plugins.GUI import Main_Window
if Main_Window.qt_application != None:
Print('Error: Check_Extensions standalone script is only supported by'
' command line calls, not from the GUI.')
return
# Set up command line arguments.
argparser = argparse.ArgumentParser(
description = 'X4 Extension Checker, part of X4 Customizer v{}'.format(Get_Version()),
epilog = 'Example: Check_Extensions.bat mymod -x4 "C:\Steam\SteamApps\common\X4 Foundations"',
)
argparser.add_argument(
'extensions',
default = None,
nargs = '*',
help = 'Optional names of an extensions to check, space separated.'
' If none given, all enabled extensions are checked.'
)
argparser.add_argument(
'-x4',
default = None,
metavar = 'Path',
nargs = 1,
help = 'Path to the X4 installation folder.'
' If not given, uses the default in Settings.')
argparser.add_argument(
'-user',
default = None,
metavar = 'Path',
nargs = 1,
help = 'Path to the user x4 documents folder.'
' If not given, uses the default in Settings.')
argparser.add_argument(
'-more_orders',
action = 'store_true',
help = 'Performs additional loading order tests, loading extensions'
' at the earliest or latest opportunity; otherwise only'
' alphabetical ordering by folder name is used.'
)
args = argparser.parse_args(sys.argv[1:])
# Copy over a couple paths to Settings; let it deal with validation
# and defaults.
if args.x4 != None:
Settings.path_to_x4_folder = args.x4
if args.user != None:
Settings.path_to_user_folder = args.user
# Include the current output extension in the check.
Settings.ignore_output_extension = False
# If no extension names given, grab all of them.
# This will also initialize the file system.
if not args.extensions:
args.extensions = File_System.Get_Extension_Names()
passed = []
failed = []
for extension in args.extensions:
if Check_Extension(extension, check_other_orderings = args.more_orders):
passed.append(extension)
else:
failed.append(extension)
# Print out a nice summary.
summary = []
for result, ext_list in [
('Passed', passed),
('Failed', failed)
]:
print()
print(result + ':')
for ext_name in ext_list:
print(' ' + ext_name)
if not ext_list:
print(' None')
# Add a little bit for a final line.
summary.append('{} {}'.format(len(ext_list), result))
print()
print('Overall: ' + ', '.join(summary))
def _Build_DockingBay_Objects():
File_System.Load_Files('*assets/props/SurfaceElements/*.xml')
game_files = File_System.Get_Asset_Files_By_Class('macros', 'dockingbay')
return Create_Objects_From_Asset_Files(game_files, dockingbay_item_macros)
def _Build_Scanner_Objects():
File_System.Load_Files('*assets/props/SurfaceElements/*.xml')
game_files = File_System.Get_Asset_Files_By_Class('macros', 'scanner')
return Create_Objects_From_Asset_Files(game_files, scanner_item_macros)
def _Build_Cockpit_Objects():
game_files = File_System.Get_All_Indexed_Files('macros', 'cockpit_*')
return Create_Objects_From_Asset_Files(game_files, [])
##############################################################################
def Patch_Shader_Files(shader_names, testmode=False):
'''
Edit asteroid shader to support transparency.
* shader_names
- Names of shader ogl files to edit.
* testmode
- Sets asteroid color to white when unfaded, black when fully faded,
for testing purposes.
'''
'''
The asteroid shader, and ogl file linking to v/vh/etc., doesn't normally
support camera fade. However, fading rules are in the common.v file,
and fade parameters can be added to the shaer ogl.
Note: ogl is xml format.
Note:
.v : vertex shader
.f : fragment shader (most logic is here)
Note: multiple ogl files can link to the same f shader file.
Ensure a) that each shader file is modified once, and b) that each ogl
file using a shader is also modified to fill in fade range defaults.
Note: some ogl files are empty; skip them (will have load_error flagged).
Note: in testing, other users of these shaders tend to error,
apparently tracing back to the V_cameraposition or IO_world_pos
values (at a guess), which even when not executed still cause a problem.
It is possible the other ogl spec files do not have corresponding
defines for the above values to be present.
Possible fixes:
- Duplicate all shaders being edited, link their ogl to the
unique versions, so original ogl/shader files are unchanged.
- Modify other ogl files to define whatever is needed to ensure
these variables are available.
- Modify the common header to ensure this vars are available.
- Regenerate these vars here (if possible).
'''
# From ogl files, get their fragment shader names.
shader_f_names = []
# Go through ogl files and uniquify/modify them.
for shader_name in shader_names:
shader_ogl_file = Load_File(f'shadergl/high_spec/{shader_name}')
# Skip empty files.
if shader_ogl_file.load_error:
continue
# Copy out the root.
xml_root = shader_ogl_file.Get_Root()
# Grab the fragment shader name.
shader_node = xml_root.find('./shader[@type="fragment"]')
shader_f_name = shader_node.get('name')
# Record it.
if shader_f_name not in shader_f_names:
shader_f_names.append(shader_f_name)
# Uniquify it.
shader_f_name = shader_f_name.replace('.f', '_faded.f')
shader_node.set('name', shader_f_name)
# Want to add to the properties list.
properties = xml_root.find('./properties')
# -Removed; these defines don't seem to work, and may be undesirable
# anyway since they might conflict with existing logic/defines,
# eg. doubling up fade multipliers.
# Add defines for the camera distance and fade.
# Put these at the top, with other defines (maybe doesn't matter).
#for field in ['ABSOLUTE_CAMERA_FADE_DISTANCE', 'FADING_CAMERA_FADE_RANGE']:
# property = etree.Element('define', name = field, value = '/*def*/')
# properties.insert(0, property)
# assert property.tail == None
# The fade calculation will use these properties.
# Assign them initial defaults in the shader (which also covers possible
# cases where the shader is used by materials that weren't customized).
# Note: These default values are not expected to ever be used.
for field, value in [('ast_camera_fade_range_start', '19900000.0'),
('ast_camera_fade_range_stop', '20000000.0')]:
property = etree.Element('float', name=field, value=value)
properties.append(property)
assert property.tail == None
# Generate a new file for the new shader spec.
File_System.Add_File(
XML_File(virtual_path=shader_ogl_file.virtual_path.replace(
'.ogl', '_faded.ogl'),
xml_root=xml_root,
modified=True))
#shader_ogl_file.Update_Root(xml_root)
# Need to update the .f file that holds the actual shader logic.
for shader_f_name in shader_f_names:
shader_f_file = Load_File(f'shadergl/shaders/{shader_f_name}')
# Do raw text editing on this.
text = shader_f_file.Get_Text()
new_text = text
# Dither based approach. If setting this to 0, experimental code
# is tried further below.
if 1:
'''
Note: these comments were made in X4 3.3.
Various attempts were made to access proper transparency through
the alpha channel, but none worked.
Observations:
- OUT_Color controls reflectivity (using low-res alt backdrop),
though in theory should be actual color.
- OUT_Color1 controls ?
- OUT_Color2 controls actual color (when used?)
- Vulkan lookups suggest transparency might depend on an alpha channel,
and would be super expensive to compute anyway.
Instead, use a dithering approach, showing more pixels as it gets closer.
Can use the "discard" command to throw away a fragment.
gl_FragCoord gives the screen pixel coordinates of the fragment.
Vanilla code divides by V_viewportpixelsize to get a Percentage coordinate,
but that should be unnecessary.
Want to identify every 1/alpha'th pixel.
Eg. alpha 0.5, want every 2nd pixel.
v * a = pix_selected + fraction
If this just rolled over, pick this pixel.
if( fract(v*a) >= a) discard;
If a == 1, fract will always be 0, 0 >= 1, so discards none.
If a == 0, fract will always be 0, 0 >= 0, so discards all.
Note: dithering on rotating asteroids creates a shimmer effect,
as lighter or darker areas of the asteroid rotate into display pixels.
Further, since dither is calculated by distance, and different points
on the asteroid are at different distances, the pixels shown are
also moving around (since asteroid isn't perfectly round).
To mitigate shimmer, also adjust the object coloring.
A couple options:
- Dim to black. This is similar to some alpha examples. Would work
well on a dark background (common in space).
- Average to the asteroid base color. This would require specifically
setting the base color for each asteroid type (brown for ore,
blueish for ice, etc.). Avoids a black->blueish transition on ice.
May look worse against a dark backdrop (eg. one blue/white pixel).
This code will go with generic color dimming for now.
Result: seems to help, though not perfect.
Further shimmer mitigation is possible by reducing volatility of
the camera depth.
In reading, it doesn't appear pixel/fragment shaders normally
support distance to the original object, only to their
particular point on the surface.
But, if the cam distance (or computed alpha) is rounded off, it
will create somewhat more stable bins. There would still be
a problem when the asteroid surface moves from one bin to another,
but most of the shimmer should be reduced.
Rounding the fade factor is probably more robust than cam distance.
Eg. fade = floor(fade * 100) / 100
Result: somewhat works, but makes the pixel selection pattern
really obvious. Where (dist.x+dist.y) is used, gets diagonal
lines of drawn pixels.
Cull pixel selection
This has a problem with creating obvious patterns.
Cull x < cuttoff || y < cuttoff:
- Get mesh of dark lines (culled points).
Cull x < cuttoff && y < cuttoff:
- Get mesh of shown lines (non-culled points).
Cull x + y < cuttoff
- Get diagonal lines.
TODO: what is a good culling formula that spreads out the points,
both when mostly faded (first few points shown) and when mostly
shown (last few points culled).
Patterns are most obvious when zooming in, but can be somewhat
seen by leaning forward, or identified when turning the camera.
Reverse-shimmer is a somewhat different problem where eg. a blue/white
ice asteroid that is mostly drawn will have some black background
pixels shimmering around and standing out.
This could be addressed by fading in with two steps:
- Dither region, where more pixels are drawn, all black.
- Color region, where pixels adjusted from black to wanted color.
Such an approach would also address other shimmer problems above,
with the caveat that it might be even more sensitive to the
overall background color (black is good, otherwise bad).
Note: small asteroids drawing in front of already-visible larger
asteroids would cause this background discrepency, eg. a small
ice chunk starts dithering in black pixels when a large ice
asteroid behind it is blue/white.
For now, ignore this problem, as solutions are potentially worse.
TODO: adjust for viewport width (or whatever it was called), so that
zoomed in views see more of the asteroid draw (reducing obviousness
of the dither effect when using a zoom hotkey).
TODO: maybe discard if not gl_FrontFacing; probably not important.
To stabalize pattern when camera turns:
Use the angle from cam pos to object pos.
angles = sin((cam.x - obj.x) / (cam.z - obj.z)) + sin((cam.y - obj.y) / (cam.z - obj.z))
Tweak this based on:
- Resolution; reduced angle between pixels at higher res.
- Round to roughly matching up to pixel density.
What is the angle between pixels?
Guess: 100 fov at 1650 wide, so 5940 pix in full circle, ~6k.
Can compute live based on V_viewportpixelsize.x, though fov
is unclear (maybe based on x/y ratio?).
In testing, centering on a nav beacon and counting turns of the
camera to get back to it, it takes 4.3 turns, or 84 degrees.
Note: trying with the 100 assumption had some dither noise when
turning the camera, so that was wrong; maybe 84 is right?
Scaling by resolution: fov = 84 / (10/16) * x/y = 52.5 * x/y
Does this even need an actual angle?
Can get the xyz vector from the camera to the object, and adjust
to be a unit vector:
dir = (cam.xyz - obj.xyz)
dir = dir / distance(dir)
That's probably good enough, but how to round for stability?
May need to use atan anyway to get angles.
Test result:
- Looks mostly stable, except some shimmering noticed occasionally,
notably near the +x axis center.
- Zooming looks bad, but not much to be done about that.
- Some shimmer when turning just due to resolution and aliasing,
similar to general object edges.
'''
# Pick number of fade stepping bins, used to reduce shimmer.
# Should be few enough that steppings don't stand out visually.
num_bins = 20
# Copy over the IO_Fade calculation from the common.v, and
# do any customization. This also allows unique var names, to
# avoid stepping on existing fade variables (eg. IO_fade).
# Note: ast_fade is live through the function in testmode, so give
# it a name likely to be unique.
new_code = f'''
float ast_cameradistance = abs(distance(V_cameraposition.xyz, IO_world_pos.xyz));
float ast_faderange = U_ast_camera_fade_range_stop - U_ast_camera_fade_range_start;
float ast_fade = 1.0 - clamp(abs(ast_cameradistance) - U_ast_camera_fade_range_start, 0.0, ast_faderange) / ast_faderange;
ast_fade = round(ast_fade * {num_bins:.1f}) / {num_bins:.1f};
'''
# Add in the discard check if not in test mode.
# Want to avoid diagonal patterns (x+y) in favor of a better scatter.
if not testmode:
#new_code += '''
# if( fract((gl_FragCoord.x + gl_FragCoord.y) * ast_fade) >= ast_fade)
# discard;
# '''
# Make a 2-wide vector for this. Note: vulkan doesn't seem to
# support something from documentation checked (sqrt? bvec?
# greaterThanEqual?), so expand out the comparison.
# Want to keep x/y crossing points, so discard unwanted x and y
# (OR the check).
#new_code += '''
# if (ast_fade < 0.999){
# float ast_fade_sqrt = sqrt(ast_fade);
# vec2 ast_factions = fract(gl_FragCoord.xy * ast_fade_sqrt);
# if( ast_factions.x >= ast_fade_sqrt || ast_factions.y >= ast_fade_sqrt)
# discard;
# }
# '''
# Better idea: use the fragment xyz, so the discard pattern doesnt
# follow the camera angle when turned.
# If the coordinate is a noisy float (eg. not 2.00000), can use
# its deep fractional part as a sort of random value.
# Result: looks good on a still object, but asteroid rotation
# creates shimmier, so reject.
#new_code += '''
# if (ast_fade < 0.999){
# float psuedo_rand = fract((IO_world_pos.x + IO_world_pos.y + IO_world_pos.z) * 16.0);
# if( psuedo_rand >= ast_fade)
# discard;
# }
# '''
# Try to create a random value from the screen xy position.
# Note: quick reading indicates gpu sin/cos is just 1 cycle.
# Example of randomizer calculation here:
# https://stackoverflow.com/questions/4200224/random-noise-functions-for-glsl
# Works decently, though dither tracks with camera rotation.
if 0:
new_code += '''
if (ast_fade < 0.999){
float psuedo_rand = fract(sin(dot(gl_FragCoord.xy, vec2(12.9898,78.233))) * 43758.5453);
if( psuedo_rand >= ast_fade)
discard;
}
'''
# More complex case: compute a rounded angle.
if 1:
# Fov is roughly 52.5 * x/y of the resolution.
# angle_per_pixel = fov / x
# Want to round based on angle_per_pixel.
# roundto = 52.5 * x/y / x = 52.5 / y
# Switch to radians.
round_y_ratio = math.pi * 2 / 360 * 52.5
new_code += f'''
if (ast_fade < 0.999){{
float roundto = {round_y_ratio} / V_viewportpixelsize.y;
vec3 dir = V_cameraposition.xyz - IO_world_pos.xyz;
// Note: atan is -pi to pi.
vec2 angles = vec2(
atan(dir.x, dir.z),
atan(dir.y, dir.z)
);
// Round them.
angles = floor((angles / roundto) + 0.5) * roundto;
// Get a rand from the angles.
// Ideally don't want patterns, eg. circles of discards,
// so don't add the angles. Can do what was done above
// with xy.
float psuedo_rand = fract(sin(dot(angles, vec2(12.9898,78.233))) * 43758.5453);
if( psuedo_rand >= ast_fade)
discard;
}}
'''
# Replace a line near the start of main, for fast discard (maybe slightly
# more performant).
# TODO: make the ref line more robust.
ref_line = 'main()\n{'
assert new_text.count(ref_line) == 1
new_text = new_text.replace(ref_line, ref_line + new_code)
# In test mode, shortcut the ast_fade to the asteroid color.
# Close asteroids will be white, far away black (ideally).
# This overwrites the normal asteroid output result.
if testmode:
new_code = '''
OUT_Color = half4(0);
OUT_Color1 = vec4(0);
OUT_Color2 = vec4(ast_fade,ast_fade,ast_fade,0);
'''
# Replace a late commented line, overriding out_color.
# TODO: more robust way to catch close of main.
ref_line = '}'
new_text = (new_code + ref_line).join(
new_text.rsplit(ref_line, 1))
# When not in test mode, dim the asteroid final color.
# TODO: does this work for all shaders, eg. color in out_color2?
# Or perhaps can all color fields (eg. reflectivity) be dimmed
# equally without impact?
# Makes everything green tinted?
if not testmode and 0:
new_code = '''
OUT_Color = OUT_Color * ast_fade;
OUT_Color1 = OUT_Color1 * ast_fade;
OUT_Color2 = OUT_Color2 * ast_fade;
'''
ref_line = '}'
new_text = (new_code + ref_line).join(
new_text.rsplit(ref_line, 1))
# Based on experimentation below, can play with the
# asteroid normal/specular to introduce some light transparency,
# which may help with a smoother fade-in alongside the dither.
# Set the normal to average between 0 rgb and (1-ast_fade)
# alpha, and what it would otherwise be, for smoother fade-in.
#
# -Removed; in practice, the norm/spec only draws when the
# asteroid is somewhat closer than its render distance, eg.
# it will be dither in 40% and then suddenly the semi-clear
# normals kick in. (Tested on medium roids in Faulty Logic.)
# TODO: maybe revisit if finding a way to extend
# normals out to further distances (may require
# messing with models or something).
if not testmode and 0:
new_code = '''
OUT_Color.a = (OUT_Color.a + 1 - ast_fade) / 2;
OUT_Color.rgb = OUT_Color.rgb * ast_fade;
//OUT_Color.a = 1.0;
//OUT_Color.rgb = half3(0.0, 0.0, 0.0);
'''
ref_line = '}'
new_text = (new_code + ref_line).join(
new_text.rsplit(ref_line, 1))
elif 0:
# Experiment with transparency, hopefully finding out how to make
# it work in x4 4.0.
'''
The F_alphascale property appears to be supplied by the game
engine. In testing, this appears to start at 0, says 0 for a
while, then snaps to 1 when halfway to the asteroid (around
when the asteroid goes fully opaque in egosoft's 4.0 handling).
Egosofts actual fade-in appears to be done separately, outside
of the shader, so this isn't useful.
The region_lodvalues.xsd file indicates that there are
two contributors to engine-calculated fade, one for time (if
asteroids spawn in inside their render distance), one for
distance. The distance version is janky in testing, particularly
in Faulty Logic, where an asteroid quickly fades in over ~50m
to ~50% transparent black, then slowly fills in color over ~30km
(slowly losing transparency), then quickly goes from 80%->100%
opaque at around 35km. Tested on a mid-sized silicon asteroid.
This shader edit will be similar to the 3.3 version, calculating
a smoother fade-in alpha, and will experiment with applying
it to the asteroid alpha channel (maybe without success).
Theory of shader operation:
a) Differred rendering is on
b) B_deferred_draw is true?
c) Shader calls macro DEFERRED_OUTPUT which sets colors,
passing it the shader computed rgb as GLOW param.
d) Macro assigns:
OUT_Color2 = ColorGlow.rgb, 3
e) Macro calls other macro, STORE_GBUFFER
f) STORE_GBUFFER assigns:
OUT_Color = NORMAL.rgb, spec (=0.2)
OUT_Color1 = BASECOLOR.rgb, METAL
Overall testing result: the egosoft transparency setting appears
to be done separately from the shader, and while shader edits
could expose some moderate transparency, side effects from
haziness just make it look bad (and cannot be hidden since
asteroids wouldnt go fully transparent).
Giving up on this path for now to go back to dithering.
'''
# Calculate the custom fade.
# See notes above (on 3.3 version) for explanation.
new_code = f'''
float ast_cameradistance = abs(distance(V_cameraposition.xyz, IO_world_pos.xyz));
float ast_faderange = U_ast_camera_fade_range_stop - U_ast_camera_fade_range_start;
float ast_fade = 1.0 - clamp(abs(ast_cameradistance) - U_ast_camera_fade_range_start, 0.0, ast_faderange) / ast_faderange;
'''
# Take the min of this and the engine supplied value.
# TODO: retest; remove for now; alphascale doesnt actually appear
# to be egosofts engine alpha.
new_code += f'''
//ast_fade = min(ast_fade, F_alphascale);
//ast_fade = F_alphascale;
ast_fade = 0.5;
'''
# Change uses of F_alphascale to instead use ast_fade later
# in the shader (should just be two uses, one for z depth, other
# for final output alpha adjustment).
# (Do this before inserting ast_fade calculation).
# -Removed for now;
#new_text = new_text.replace('F_alphascale', 'ast_fade')
# Replace a line near the start of main, so it is in place before
# first use of ast_fade.
# TODO: make the ref line more robust.
ref_line = 'main()\n{'
assert new_text.count(ref_line) == 1
new_text = new_text.replace(ref_line, ref_line + new_code)
# Apply to the final asteroid color.
# TODO: does this work for all shaders, eg. color in out_color2?
# Or perhaps can all color fields (eg. reflectivity) be dimmed
# equally without impact?
# Note: DEFERRED_OUTPUT appears to be in use, as in testing
# a full reapplication of the last OUT_Color line just before
# the function end causes oddities in asteroid appearance (they
# show up half transparent at any range).
new_code = '''
//OUT_Color = OUT_Color * ast_fade;
//OUT_Color1 = OUT_Color1 * ast_fade;
//OUT_Color2 = OUT_Color2 * ast_fade;
// Does nothing.
//OUT_Color.a = 0.5;
//OUT_Color.a = OUT_Color.a * ast_fade;
//-Causes half transparent asteroids, any distance; background
// is blurred through asteroid.
// Asteroids also take on a haziness at a distance (seem to be
// taking on some of the average background color), so they
// show up dark against light colored background, and light
// against dark colored background.
//OUT_Color = half4(finalColor.rgb, ColorBaseDiffuse.a * F_alphascale);
// Similar but less transparent.
//OUT_Color = 0.5 * half4(finalColor.rgb, ColorBaseDiffuse.a * F_alphascale);
// Distant asteroids slightly white/shiny; generally more transparent.
//OUT_Color = 2 * half4(finalColor.rgb, ColorBaseDiffuse.a * F_alphascale);
// Similar to above, distant slightly white (maybe background
// showing through? but whiter than background, perhaps due
// to blurred background).
//OUT_Color = half4(finalColor.rgb, ColorBaseDiffuse.a * F_alphascale * 2);
// Asteroid seems a bit desaturated and bland, but opaque.
//OUT_Color.rgb = finalColor.rgb;
// Doing rgb and a separately has the same effect as together,
// so no oddities with half4.
//OUT_Color.rgb = finalColor.rgb;
//OUT_Color.a = ColorBaseDiffuse.a * F_alphascale;
// Opaque but dark asteroids (not black though).
//OUT_Color = half4(0,0,0,0);
// Opaque, distant asteroids white, nearby colored normally;
// zooming in switches white roids to normal (so something
// to do with rendering angle and not distance).
//OUT_Color = half4(0.5,0.5,0.5,0.5);
// Opaque, distant asteroids white again, similar to above.
//OUT_Color.rgb = half3(0.5,0.5,0.5);
// Alpha changes did nothing notable.
//OUT_Color.a = 0.5;
//OUT_Color.a = 0.05;
//OUT_Color.a = 0.95;
//OUT_Color.a = ColorBaseDiffuse.a * F_alphascale;
// Distant asteroids are lighter colored, normal up close.
//OUT_Color.a = 10;
// Distant asteroids even lighter/glowier with blue halo,
// normal up close.
//OUT_Color.a = 128;
// Opaque and extremely shiny at distance, normal up close.
//OUT_Color.rgb = half3(128,128,128);
// Distant roids shiny red/green instead of white.
//OUT_Color.r = 128;
//OUT_Color.g = 128;
// Just try to rescale alpha, to narrow down its number.
//OUT_Color.rgb = finalColor.rgb;
// Opaque to stars, seemingly, but background fog/nebula
// show through slightly.
//OUT_Color.a = 0.1;
// Modestly transparant, can make out the blurred stars.
//OUT_Color.a = 0.5;
// More transparant, but feels like 80%; can still make out
// the asteroid, and distance asteroids still have a light
// colored haze to them.
//OUT_Color.a = 1.0;
// Similar to above.
//OUT_Color.a = 1.5;
// Try to rescale rgb, to narrow down its number.
//OUT_Color.a = 1.0;
// Around 80% transparent still, but asteroid has a darker hue.
//OUT_Color.rgb = half3(0.0, 0.0, 0.0);
// Asteroids on one half of the sector are bright white,
// at any distance, based on player view point (eg. can strafe
// to switch an asteroid from normal to white gradually).
//OUT_Color.rgb = half3(1.0, 1.0, 1.0);
// Giving up here; it might be negative or something, but
// seems like a dead end with the haziness.
// Try out the other color alphas.
// No noticed effect from this.
//OUT_Color2.a = 0.5;
// Nor from this.
//OUT_Color1.a = 0.5;
'''
ref_line = '}'
new_text = (new_code + ref_line).join(new_text.rsplit(ref_line, 1))
# In test mode, shortcut the ast_fade to the asteroid color.
# Close asteroids will be white, far away black (ideally).
# This overwrites the normal asteroid output result.
if testmode:
new_code = '''
OUT_Color = half4(0);
OUT_Color1 = vec4(0);
OUT_Color2 = vec4(ast_fade,ast_fade,ast_fade,0);
'''
# Replace a late commented line, overriding out_color.
# TODO: more robust way to catch close of main.
ref_line = '}'
new_text = (new_code + ref_line).join(
new_text.rsplit(ref_line, 1))
# Uniquify the file.
File_System.Add_File(
Text_File(virtual_path=shader_f_file.virtual_path.replace(
'.f', '_faded.f'),
text=new_text,
modified=True))
return
def Adjust_Mission_Reward_Mod_Chance(new_chance=2, ):
'''
Adjusts generic mission chance to reward a mod instead of credits.
The vanilla chance is 2% for a mod, 98% for credits.
Pending update for x4 3.0+.
* mod_chance
- Int, the new percent chance of rewarding a mod.
- Should be between 0 and 100.
'''
'''
Many generic missions repeat this bit of code:
<do_any>
<do_if value="not $RewardCr" weight="98">
...
</do_if>
<do_if value="not $RewardObj" weight="2">
...
</do_if>
</do_any>
Can hunt down the appropriate nodes to make the edit.
Update: X4 3.0 adjusts all of the missions to have a seminar reward
as well, and they vary in chances (90/2/8, 91/2/7, etc.).
'''
# Validate the mod_chance range.
mod_chance = int(new_chance)
if mod_chance < 0:
mod_chance = 0
elif mod_chance > 100:
mod_chance = 100
# Want the various generic mission scripts.
# These all are prefixed with GM_
game_files = File_System.Load_Files('md/GM_*')
for game_file in game_files:
xml_root = game_file.Get_Root()
# Find the parent reward node.
reward_nodes = xml_root.xpath('//do_if[@value="$GenerateReward"]')
if len(reward_nodes) != 1:
continue
# Pick out the two nodes of interest.
high_nodes = reward_nodes[0].xpath(
'do_any/do_if[@value="not $RewardCr"][@weight="98"]')
low_nodes = reward_nodes[0].xpath(
'do_any/do_if[@value="not $RewardObj"][@weight="2"]')
# 1 of each should have been found.
if len(high_nodes) != 1 or len(low_nodes) != 1:
continue
# Make the edits.
high_nodes[0].set('weight', str(100 - mod_chance))
low_nodes[0].set('weight', str(mod_chance))
game_file.Update_Root(xml_root)
return
