def collect_data(asset: UAsset) -> Tuple[str, Any]:
if args.default and args.export is not None:
print("Cannot specify an export with --default", file=sys.stderr)
sys.exit(1)
if args.default:
# Produce default export only
if not hasattr(asset, 'default_export'):
print("Asset does not have a default export", file=sys.stderr)
sys.exit(1)
assert asset.default_export and asset.default_export.fullname
data = sanitise_output(asset.default_export.properties)
filename = create_filename(asset.default_export.fullname)
elif args.export:
# Single export
as_int: Optional[int] = None
try:
as_int = int(args.export)
except ValueError:
pass
if as_int is not None:
# Integer-based export lookup
if as_int < 0 or as_int >= len(asset.exports.values):
print(f"Maximum export index for this asset is {len(asset.exports.values)-1}", file=sys.stderr)
sys.exit(1)
export = asset.exports[as_int]
else:
# Name-based export lookup
found_indexes = []
search_name = args.export.lower()
for i in range(len(asset.exports.values)):
export = asset.exports.values[i]
if str(export.name).lower() == search_name:
found_indexes.append(i)
if found_indexes:
print("Export with this name not found", file=sys.stderr)
sys.exit(1)
elif len(found_indexes) > 1:
print("This name was found at multiple indexes:", file=sys.stderr)
pprint(found_indexes, stream=sys.stderr)
sys.exit(1)
export = asset.exports.values[found_indexes[0]]
data = sanitise_output(export.properties)
filename = create_filename(export.fullname)
else:
# Full asset extraction
data = sanitise_output(asset)
assert asset.assetname
filename = create_filename(asset.assetname)
return (filename, data)
def convert_npc_remaps(pgd: UAsset) -> List[ClassRemap]:
assert pgd.default_export
export_data = pgd.default_export.properties
npcs = export_data.get_property('Remap_NPC', fallback=None)
containers = export_data.get_property('Remap_NPCSpawnEntries',
fallback=None)
remaps = []
if npcs:
remaps += npcs.values
if containers:
remaps += containers.values
out = []
for entry in remaps:
d = entry.as_dict()
# Get the class-to-remap and ensure it is a valid reference.
# Skip otherwise, as nulls cannot be spawned.
from_class = d.get('FromClass', None)
if not from_class:
continue
# Push the remap to the output list.
v = ClassRemap(
from_bp=sanitise_output(from_class),
to=sanitise_output(d.get('ToClass', None)),
)
out.append(v)
return out
def convert_single_class_swap(d):
result = WeighedClassSwap(from_class=sanitise_output(d['FromClass']),
exact=bool(d.get('bExactMatch', True)),
to=sanitise_output(d['ToClasses']),
weights=d['Weights'].values)
if d['ActiveEvent'] and d['ActiveEvent'].value and d[
'ActiveEvent'].value.value:
# Assigning "None" here is safe as it is the field default and therefore omitted
result.during = str(d['ActiveEvent'])
return result
def segregate_container_additions(
pgd: UAsset) -> Optional[List[RuntimeGroupAddition]]:
if not pgd.default_export:
return None
export_data = pgd.default_export.properties
d = export_data.get_property('TheNPCSpawnEntriesContainerAdditions',
fallback=None)
if not d:
return None
# Extract the addition entries
changes: Dict[str, List[RuntimeGroupAddition]] = defaultdict(list)
for add in d.values:
add = add.as_dict()
klass = add['SpawnEntriesContainerClass']
entries = add['AdditionalNPCSpawnEntries'].values
limits = add['AdditionalNPCSpawnLimits'].values
if not klass.value.value or (not entries and not limits):
continue
out = RuntimeGroupAddition(
bp=sanitise_output(klass),
entries=[convert_group_entry(entry) for entry in entries],
limits=convert_limit_entries(limits),
)
# Skip if no data
if not out.limits and not out.entries:
continue
# Append to the fragment list
changes[out.bp].append(out)
return _merge_changes(changes)
def main():
arkman = ArkSteamManager()
loader = arkman.getLoader()
config = get_global_config()
assetname = sys.argv[1] if len(sys.argv) > 1 else None
if not assetname:
print('Usage: python ueexport.py <assetname>')
sys.exit(1)
assetname = find_asset(assetname, loader)
if not assetname:
print("Not found")
sys.exit(1)
asset = loader[assetname]
assert asset.default_export
if not asset.default_export:
print("Asset has no default export")
sys.exit(2)
export = asset.default_export
data = sanitise_output(export.properties)
pprint(data)
save_as_json(data, f'output/{asset.name}.{export.name}.json', pretty=True)
def _extract_and_save(self, version: str, modid: Optional[str], base_path: Path, relative_path: PurePosixPath,
proxy_iter: Iterator[UEProxyStructure]):
# Work out the output path (cleaned)
clean_relative_path = PurePosixPath('/'.join(get_valid_filename(p) for p in relative_path.parts))
output_path = Path(base_path / clean_relative_path)
# Setup the output structure
results: List[Any] = []
format_version = self.get_format_version()
output: Dict[str, Any] = dict(version=version, format=format_version)
# Pre-data comes before the main items
pre_data = self.get_pre_data(modid) or dict()
pre_data = sanitise_output(pre_data)
output.update(pre_data)
# Main items array
output[self.get_field()] = results
# Do the actual export into the existing `results` list
for proxy in proxy_iter:
item_output = self.extract(proxy)
if item_output:
item_output = sanitise_output(item_output)
results.append(item_output)
# Make the results available to get_post_data
self.gathered_results = results
# Post-data comes after the main items
post_data = self.get_post_data(modid) or {}
post_data = sanitise_output(post_data)
output.update(post_data)
post_data_has_content = post_data and any(post_data.values())
# Clear gathered data reference
del self.gathered_results
# Save if the data changed
if results or post_data_has_content:
save_json_if_changed(output, output_path, self.get_use_pretty())
else:
# ...but remove an existing one if the output was empty
if output_path.is_file():
output_path.unlink()
def extract(cls, proxy: UEProxyStructure) -> GatheringResult:
volume: TogglePainVolume = cast(TogglePainVolume, proxy)
box = get_volume_bounds(volume)
return models.PainVolume(
start=box.start,
center=box.center,
end=box.end,
immune=sanitise_output(volume.get('ActorClassesToExclude', fallback=[])),
)
def extract(cls, proxy: UEProxyStructure) -> GatheringResult:
dispatcher: MissionDispatcher_MultiUsePylon = cast(MissionDispatcher_MultiUsePylon, proxy)
type_id = dispatcher.MissionTypeIndex[0].value
location = get_actor_location_vector(dispatcher)
return models.MissionDispatcher(type_=cls.MISSION_TYPE_MAP.get(type_id, type_id),
missions=sanitise_output(dispatcher.MissionTypes[0].values),
x=location.x,
y=location.y,
z=location.z)
def _zip_swap_outputs(
d: Dict[str, Any]) -> Iterable[Tuple[float, Optional[str]]]:
npcs: ArrayProperty = d['ToClasses']
weights = d['Weights'].values
num_weights = len(weights)
for index, kls in enumerate(npcs.values):
# Get weight of this class. Defaults to 1 if array is too short.
weight = float(weights[index]) if index < num_weights else 1.0
yield (weight, sanitise_output(kls))
def ingest_level(self, level: UAsset):
assert level.assetname
assert level.loader
assetname = level.assetname
loader = cast(AssetLoader, level.loader)
# Check if asset is a persistent level and mark it as such in map info object
if not getattr(level, 'tile_info', None) and self.persistent_level != assetname:
if self.persistent_level:
logger.warning(f'Found a persistent level ({assetname}), but {self.persistent_level} was located earlier')
else:
self.persistent_level = assetname
# Go through each export and, if valuable, gather data from it.
for export in level.exports:
gatherer = find_gatherer_for_export(export)
if not gatherer:
continue
# Extract data using gatherer class.
try:
data = gatherer.extract(proxy=gather_properties(export))
except Exception:
logger.warning(f'Gathering properties failed for export "{export.name}" in {assetname}', exc_info=True)
continue
# Add fragment to data lists
if data:
if isinstance(data, GeneratorType):
data_fragments: list = sanitise_output(list(data))
for fragment in data_fragments:
if fragment:
self.data[gatherer].append(fragment)
else:
fragment = sanitise_output(data)
self.data[gatherer].append(fragment)
# Preemptively remove the level from linker cache.
loader.cache.remove(assetname)
def convert_single_class_swap(d: Dict[str, Any]) -> Optional[WeighedClassSwap]:
result = WeighedClassSwap(from_class=sanitise_output(d['FromClass']),
exact=bool(d.get('bExactMatch', True)),
to=list(_zip_swap_outputs(d)))
if not result.from_class:
return None
if d['ActiveEvent'] and d['ActiveEvent'].value and d[
'ActiveEvent'].value.value:
# Assigning "None" here is safe as it is the field default and therefore omitted
result.during = str(d['ActiveEvent'])
return result
def convert_group_entry(struct) -> NpcGroup:
d = struct.as_dict()
out = NpcGroup(
name=str(d['AnEntryName']),
weight=d['EntryWeight'],
species=list(),
randomSwaps=[],
)
# Export zipped NPC entries
chances = d['NPCsToSpawnPercentageChance'].values
offsets = d['NPCsSpawnOffsets'].values
num_chances = len(chances)
num_offsets = len(offsets)
for index, kls in enumerate(d['NPCsToSpawn'].values):
# Ensure the NPC class is not null.
# The bp field was marked as optional before, but it left dead entries with only a chance.
# Since we know more about NPC spawning now, and AE 893735676 has shipped a very dirty update
# around Sep 10th, 2021, skip those entries entirely.
if kls and kls.value and kls.value.value:
npc = NpcEntry(
chance=chances[index] if index < num_chances else 1,
bp=sanitise_output(kls),
offset=sanitise_output(offsets[index] if index < num_offsets
else Vector(x=0, y=0, z=0)),
)
out.species.append(npc)
# Export local random class swaps if any exist
swaps = d['NPCRandomSpawnClassWeights'].values
for entry in swaps:
rule = convert_single_class_swap(entry.as_dict())
if rule:
out.randomSwaps.append(rule)
return out
def convert_limit_entries(array) -> Iterable[NpcLimit]:
already_found = set()
for entry in array:
d = entry.as_dict()
npc_class = sanitise_output(d['NPCClass'])
mult = d['MaxPercentageOfDesiredNumToAllow']
# We've already seen this class so this rule does not matter in context of this container, skip it.
if npc_class in already_found:
continue
# Only yield if the NPC class isn't a null and the max multiplier isn't 1.0.
if npc_class and mult != 1.0:
already_found.add(npc_class)
yield NpcLimit(bp=npc_class, mult=mult)
def _gather_data_from_levels(self, levels: List[str], known_persistent: Optional[str] = None) -> MapInfo:
'''
Goes through each sublevel, gathering data and looking for the persistent level.
'''
map_info = MapInfo(data=dict())
for assetname in levels:
asset = self.manager.loader[assetname]
# Check if asset is a persistent level and collect data from it.
if not getattr(asset, 'tile_info', None) and (not known_persistent or known_persistent == assetname):
if getattr(map_info, 'persistent_level', None):
logger.warning(
f'Found another persistent level ({assetname}), but {map_info.persistent_level} was located earlier: skipping.'
)
continue
map_info.persistent_level = assetname
# Go through each export and, if valuable, gather data from it.
for export in asset.exports:
helper = find_gatherer_for_export(export)
if helper:
# Make sure the data list is initialized.
category_name = helper.get_category_name()
if category_name not in map_info.data:
map_info.data[category_name] = list()
# Extract data using helper class.
try:
for data_fragment in helper.extract(proxy=gather_properties(export)):
if not data_fragment:
continue
# Sanitise the data fragment to remove references to the UE tree.
data_fragment = sanitise_output(data_fragment)
# Add to the list.
map_info.data[category_name].append(data_fragment)
except: # pylint: disable=bare-except
logger.warning(f'Gathering properties failed for export "{export.name}" in {assetname}', exc_info=True)
continue
# Preemptively remove the level from linker cache.
self.manager.loader.cache.remove(assetname)
return map_info
def extract(cls, proxy: UEProxyStructure) -> models.WorldSettings:
settings: PrimalWorldSettings = cast(PrimalWorldSettings, proxy)
source: ExportTableItem = cast(ExportTableItem, proxy.get_source())
display_name: Union[StringProperty, str]
if settings.has_override('Title'):
display_name = settings.Title[0]
else:
display_name = get_leaf_from_assetname(source.asset.assetname)
display_name = display_name.rstrip('_P')
display_name = uelike_prettify(display_name)
result = models.WorldSettings(
source=source.asset.assetname,
name=display_name,
# Geo
latOrigin=settings.LatitudeOrigin[0],
longOrigin=settings.LongitudeOrigin[0],
latScale=settings.LatitudeScale[0],
longScale=settings.LongitudeScale[0],
# Gameplay Settings
maxDifficulty=settings.OverrideDifficultyMax[0],
mapTextures=models.InGameMapTextureSet(
held=sanitise_output(
settings.get('OverrideWeaponMapTextureFilled', 0, None)),
emptyHeld=sanitise_output(
settings.get('OverrideWeaponMapTextureEmpty', 0, None)),
empty=sanitise_output(
settings.get('OverrideUIMapTextureEmpty', 0, None)),
big=sanitise_output(
settings.get('OverrideUIMapTextureFilled', 0, None)),
small=sanitise_output(
settings.get('OverrideUIMapTextureSmall', 0, None)),
),
# Spawns
onlyEventGlobalSwaps=bool(
settings.bPreventGlobalNonEventSpawnOverrides[0]),
randomNPCClassWeights=list(cls._convert_class_swaps(settings)),
# Uploads
allowedDinoDownloads=sanitise_output(
settings.get('AllowDownloadDinoClasses', 0, ())),
)
# Calculate remaining geo fields
result.latMulti = result.latScale * 10
result.latShift = -result.latOrigin / result.latMulti
result.longMulti = result.longScale * 10
result.longShift = -result.longOrigin / result.longMulti
return result
def test_field_prop_in_constructor(field_type, field_name, value, target):
'''
Ensure UE values in models are accepted and converted
correctly, when set in the constructor.
'''
# Setup
create_fn = getattr(field_type, 'create')
field_value = create_fn(value)
kwargs = {field_name: field_value}
# Create model with field in constructor
model = UETypedModel(**kwargs)
# Convert model to dict, verify field is untouched
output = model.dict()
assert output[field_name] is field_value
# Sanitise model, verify field is converted as expected
result = sanitise_output(output)
field_result = result[field_name]
assert field_result == target
def extract(cls, proxy: UEProxyStructure) -> GatheringResult:
dispatcher: MissionDispatcher = cast(MissionDispatcher, proxy)
location = get_actor_location_vector(dispatcher)
out = models.MissionDispatcher(x=location.x,
y=location.y,
z=location.z)
# Genesis 1 has dispatchers that only allow specific missions.
if isinstance(dispatcher, MissionDispatcher_MultiUsePylon):
dispatcher_gen1 = cast(MissionDispatcher_MultiUsePylon, dispatcher)
type_id = dispatcher_gen1.MissionTypeIndex[0].value
out.type_ = cls.MISSION_TYPE_MAP.get(type_id, type_id)
out.missions = sanitise_output(
dispatcher_gen1.MissionTypes[0].values)
# Export a flag in case this dispatcher is used to start the Rockwell Prime fight.
# Allows for easier identification.
if isinstance(dispatcher, MissionDispatcher_FinalBattle):
out.isRockwellBattle = True
return out
def test_field_prop_as_attr(field_type, field_name, value, target):
'''
Ensure UE values in models are accepted and converted
correctly, when set after creation.'''
# Setup
model = UETypedModel()
create_fn = getattr(field_type, 'create')
field_value = create_fn(value)
# Set the model's field
setattr(model, field_name, field_value)
assert getattr(model, field_name) is field_value
# Convert model to dict, verify field is untouched
output = model.dict()
assert output[field_name] is field_value
# Sanitise model, verify field is converted as expected
result = sanitise_output(output)
field_result = result[field_name]
assert field_result == target
def extract(self, proxy: UEProxyStructure) -> Engram:
engram: PrimalEngramEntry = cast(PrimalEngramEntry, proxy)
out = Engram(
description=engram.ExtraEngramDescription[0]
if engram.has_override('ExtraEngramDescription') else None,
blueprintPath=engram.get_source().fullname,
itemBlueprintPath=sanitise_output(
engram.get('BluePrintEntry', 0, None)),
group=convert_engram_group(engram),
requirements=EngramRequirements(
characterLevel=engram.RequiredCharacterLevel[0],
engramPoints=engram.RequiredEngramPoints[0],
),
manualUnlock=bool(engram.bCanBeManuallyUnlocked[0]),
givesBP=bool(engram.bGiveBlueprintToPlayerInventory[0]),
)
if 'EngramRequirementSets' in engram:
out.requirements.otherEngrams = list(
convert_requirement_sets(engram))
return out
def extract(cls, proxy: UEProxyStructure) -> GatheringResult:
manager: NPCZoneManager = cast(NPCZoneManager, proxy)
# Sanity checks
spawn_group = manager.get('NPCSpawnEntriesContainerObject', 0, None)
count_volumes = manager.get('LinkedZoneVolumes', 0, None)
if not spawn_group or not spawn_group.value.value or not count_volumes:
return None
# Export properties
result = models.NPCManager(
disabled=not manager.bEnabled[0],
spawnGroup=sanitise_output(spawn_group),
minDesiredNumberOfNPC=manager.MinDesiredNumberOfNPC[0],
neverSpawnInWater=manager.bNeverSpawnInWater[0],
forceUntameable=manager.bForceUntameable[0],
)
# Export dino counting regions
result.locations = list(cls._extract_counting_volumes(count_volumes))
# Export spawn points if present
spawn_points = manager.get('SpawnPointOverrides', 0, None)
spawn_volumes = manager.get('LinkedZoneSpawnVolumeEntries', 0, None)
if spawn_points:
result.spawnPoints = list(cls._extract_spawn_points(spawn_points))
# Export spawn regions if present
# Behaviour verified in DevKit. Dinos don't spawn in spawning volumes if
# points were manually specified.
elif spawn_volumes:
result.spawnLocations = list(
cls._extract_spawn_volumes(spawn_volumes))
# Check if we extracted any spawn data at all, otherwise we can skip it.
if not result.spawnPoints and not result.spawnLocations:
return None
return result
def _convert_crate_classes(cls, entries) -> Iterable[models.ObjectPath]:
for entry in entries.values:
klass = entry.as_dict()['CrateTemplate']
if klass:
yield sanitise_output(klass)
def _extract_and_save(self,
version: str,
modid: Optional[str],
base_path: Path,
relative_path: PurePosixPath,
proxy_iter: Iterator[UEProxyStructure],
*,
schema_file: Optional[PurePosixPath] = None):
# Work out the output path (cleaned)
clean_relative_path = PurePosixPath(*(get_valid_filename(p) for p in relative_path.parts))
output_path = Path(base_path / clean_relative_path)
# Setup the output structure
results: List[Any] = []
format_version = self.get_format_version()
output: Dict[str, Any] = dict()
if schema_file:
model = self.get_schema_model() # pylint: disable=assignment-from-none # stupid pylint
assert model
expected_subtype = _get_model_list_field_type(model, self.get_field())
output['$schema'] = str(_calculate_relative_path(clean_relative_path, schema_file))
output['version'] = version
output['format'] = format_version
# Pre-data comes before the main items
pre_data = self.get_pre_data(modid) or dict()
pre_data = sanitise_output(pre_data)
output.update(pre_data)
# Main items array
output[self.get_field()] = results
# Do the actual export into the existing `results` list
for proxy in proxy_iter:
item_output = self.extract(proxy)
if item_output:
if schema_file and expected_subtype and not isinstance(item_output, expected_subtype):
raise TypeError(f"Expected {expected_subtype} from schema-enabled exported item but got {type(item_output)}")
item_output = sanitise_output(item_output)
results.append(item_output)
# Make the results available to get_post_data
self.gathered_results = results
# Post-data comes after the main items
post_data = self.get_post_data(modid) or {}
post_data = sanitise_output(post_data)
output.update(post_data)
post_data_has_content = post_data and any(post_data.values())
# Clear gathered data reference
del self.gathered_results
# Save if the data changed
if results or post_data_has_content:
save_json_if_changed(output, output_path, self.get_use_pretty())
else:
# ...but remove an existing one if the output was empty
if output_path.is_file():
output_path.unlink()
def collect_data(asset: UAsset) -> Tuple[str, Any]:
if args.default and args.export is not None:
print("Cannot specify an export with --default", file=sys.stderr)
sys.exit(1)
if args.ovi:
if not args.export and not args.default:
# Grab the default export since we need a starting point for the proxy.
args.default = True
export = find_export(asset)
assert export
assert export.fullname
if not inherits_from(export, PrimalItem.get_ue_type()):
print(f"Export {export.name} is not a subclass of PrimalItem.", file=sys.stderr)
sys.exit(1)
proxy: UEProxyStructure = gather_properties(export)
item = cast(PrimalItem, proxy)
if 'ItemIconMaterialParent' not in item:
print("Item does not use an icon shader", file=sys.stderr)
sys.exit(1)
name = get_item_name(item) or export.name
data: Dict[str, Any] = dict(
Format='2.0',
Name=name,
BlueprintPath=export.fullname,
)
assert asset.loader
mat_instance = asset.loader.load_related(item.ItemIconMaterialParent[0]).default_export
assert mat_instance
mat_properties = mat_instance.properties.as_dict()
# Convert all parameters from the material instance.
parameters = dict()
if 'ScalarParameterValues' in mat_properties:
for param in mat_properties['ScalarParameterValues'][0].values:
param_info = param.as_dict()
param_name = sanitise_output(param_info['ParameterName'])
parameters[param_name] = param_info['ParameterValue']
if 'VectorParameterValues' in mat_properties:
for param in mat_properties['VectorParameterValues'][0].values:
param_info = param.as_dict()
param_name = sanitise_output(param_info['ParameterName'])
parameters[param_name] = param_info['ParameterValue'].values[0]
if 'TextureParameterValues' in mat_properties:
for param in mat_properties['TextureParameterValues'][0].values:
param_info = param.as_dict()
param_name = sanitise_output(param_info['ParameterName'])
parameters[param_name] = param_info['ParameterValue']
if parameters:
data['2DMaterial'] = dict(Parent=mat_properties['Parent'][0], **parameters)
else:
# Export only the parent material as the instance has no parameters.
data['2DMaterial'] = mat_properties['Parent'][0]
data = sanitise_output(data)
filename = create_filename(export.fullname)
elif args.default or args.export:
export = find_export(asset)
assert export
assert export.fullname
data = sanitise_output(export.properties)
filename = create_filename(export.fullname)
else:
# Full asset extraction
data = sanitise_output(asset)
assert asset.assetname
filename = create_filename(asset.assetname)
return (filename, data)
def do_extract(root: str, excludes: Set[str]):
base_dir = output_dir / format
with ue_parsing_context(properties=True, bulk_data=False):
asset_iterator = loader.find_assetnames(
root,
exclude=excludes,
extension=ue.hierarchy.asset_extensions,
return_extension=True)
total_files = 0
total_dirs = 0
total_bytes = 0
prev_path = ''
for (assetname, _) in asset_iterator:
# print(assetname)
show_stats = False
total_files += 1
output_filename = create_filename(assetname, ext)
output_path = base_dir / output_filename
output_path.parent.mkdir(exist_ok=True, parents=True)
print(f'► {output_path}')
if str(output_path.parent) != prev_path:
total_dirs += 1
prev_path = str(output_path.parent)
print(output_path.parent.relative_to(base_dir))
if (total_dirs % 100) == 0:
show_stats = True
try:
# Skip existing files for
if output_path.is_file():
continue
try:
asset = loader[assetname]
except Exception:
print(f'ERROR: Unable to parse: {assetname}')
continue
loader.wipe_cache() # No caching!
data = sanitise_output(asset.exports)
binary = fmt(data, asset.names.values)
output_path.write_bytes(binary)
finally:
if output_path.is_file():
total_bytes += output_path.stat().st_size
if show_stats:
print()
print(f'Total dirs: {total_dirs:,}')
print(f'Total files: {total_files:,}')
print(f'Total bytes: {total_bytes/1024/1024:,.1f}Mo')
print()
print()
print('Grand total:')
print()
print(f'Total dirs: {total_dirs:,}')
print(f'Total files: {total_files:,}')
print(f'Total bytes: {total_bytes/1024/1024:,.1f}Mo')
def convert_requirement_sets(
engram: PrimalEngramEntry) -> Iterable[Optional[str]]:
for struct in engram.EngramRequirementSets[0].values:
for entry in struct.get_property('EngramEntries').values:
yield sanitise_output(entry)
