def merge_sarcs(file_name: str, sarcs: List[Union[Path, bytes]]) -> (str, bytes):
opened_sarcs: List[sarc.SARC] = []
if isinstance(sarcs[0], Path):
for i, sarc_path in enumerate(sarcs):
sarcs[i] = sarc_path.read_bytes()
for sarc_bytes in sarcs:
sarc_bytes = util.unyaz_if_needed(sarc_bytes)
try:
opened_sarcs.append(sarc.SARC(sarc_bytes))
except ValueError:
continue
all_files = {key for open_sarc in opened_sarcs for key in open_sarc.list_files()}
nested_sarcs = {}
new_sarc = sarc.SARCWriter(be=True)
files_added = []
# for file in all_files:
# dm_cache = util.get_master_modpack_dir() / 'logs' / 'dm' / file
# if dm_cache.exists():
# file_data = dm_cache.read_bytes()
# new_sarc.add_file(file, file_data)
# files_added.append(file)
for opened_sarc in reversed(opened_sarcs):
for file in [file for file in opened_sarc.list_files() if file not in files_added]:
data = opened_sarc.get_file_data(file).tobytes()
if util.is_file_modded(file.replace('.s', '.'), data, count_new=True):
if not Path(file).suffix in util.SARC_EXTS:
new_sarc.add_file(file, data)
files_added.append(file)
else:
if file not in nested_sarcs:
nested_sarcs[file] = []
nested_sarcs[file].append(util.unyaz_if_needed(data))
for file, sarcs in nested_sarcs.items():
merged_bytes = merge_sarcs(file, sarcs)[1]
if Path(file).suffix.startswith('.s') and not file.endswith('.sarc'):
merged_bytes = util.compress(merged_bytes)
new_sarc.add_file(file, merged_bytes)
files_added.append(file)
for file in [file for file in all_files if file not in files_added]:
for opened_sarc in [open_sarc for open_sarc in opened_sarcs \
if file in open_sarc.list_files()]:
new_sarc.add_file(file, opened_sarc.get_file_data(file).tobytes())
break
if 'Bootup.pack' in file_name:
for merger in [merger() for merger in mergers.get_mergers() if merger.is_bootup_injector()]:
inject = merger.get_bootup_injection()
if not inject:
continue
file, data = inject
try:
new_sarc.delete_file(file)
except KeyError:
pass
new_sarc.add_file(file, data)
return (file_name, new_sarc.get_bytes())
def _get_sizes_in_sarc(file: Union[Path, sarc.SARC]) -> {}:
calc = rstb.SizeCalculator()
sizes = {}
guess = util.get_settings_bool('guess_merge')
if isinstance(file, Path):
with file.open('rb') as s_file:
file = sarc.read_file_and_make_sarc(s_file)
if not file:
return {}
for nest_file in file.list_files():
canon = nest_file.replace('.s', '.')
data = util.unyaz_if_needed(file.get_file_data(nest_file).tobytes())
ext = Path(canon).suffix
if util.is_file_modded(canon, data) and ext not in RSTB_EXCLUDE_EXTS and canon not in RSTB_EXCLUDE_NAMES:
size = calc.calculate_file_size_with_ext(
data,
wiiu=True,
ext=ext
)
if ext == '.bdmgparam':
size = 0
if size == 0 and guess:
if ext in util.AAMP_EXTS:
size = guess_aamp_size(data, ext)
elif ext in ['.bfres', '.sbfres']:
size = guess_bfres_size(data, canon)
sizes[canon] = size
if util.is_file_sarc(nest_file) and not nest_file.endswith('.ssarc'):
try:
nest_sarc = sarc.SARC(data)
except ValueError:
continue
sizes.update(_get_sizes_in_sarc(nest_sarc))
return sizes
def _get_sizes_in_sarc(
file: Union[Path, oead.Sarc], guess: bool, is_aoc: bool = False
) -> {}:
sizes = {}
if isinstance(file, Path):
is_aoc = util.get_dlc_path() in file.as_posix()
try:
file = oead.Sarc(util.unyaz_if_needed(file.read_bytes()))
except (RuntimeError, oead.InvalidDataError):
print(f"{file} could not be opened")
return {}
for nest_file, data in [(file.name, file.data) for file in file.get_files()]:
canon = nest_file.replace(".s", ".")
if data[0:4] == b"Yaz0":
data = util.decompress(data)
ext = Path(canon).suffix
if (
util.is_file_modded(canon, data)
and ext not in EXCLUDE_EXTS
and canon not in EXCLUDE_NAMES
):
sizes[canon] = calculate_size(canon, data, guess=guess)
if ext in util.SARC_EXTS - SARC_EXCLUDES:
try:
nest_sarc = oead.Sarc(data)
except (ValueError, RuntimeError, oead.InvalidDataError):
continue
sizes.update(_get_sizes_in_sarc(nest_sarc, guess, is_aoc=is_aoc))
del nest_sarc
del data
del file
return sizes
def _convert_actorpack(actor_pack: Path, to_wiiu: bool) -> Union[None, str]:
error = None
sarc = oead.Sarc(util.unyaz_if_needed(actor_pack.read_bytes()))
new_sarc = oead.SarcWriter.from_sarc(sarc)
new_sarc.set_endianness(oead.Endianness.Big if to_wiiu else oead.Endianness.Little)
for file in sarc.get_files():
if "Physics/" in file.name and "Actor/" not in file.name:
ext = file.name[file.name.rindex(".") :]
if ext in NO_CONVERT_EXTS:
if not util.is_file_modded(
util.get_canon_name(file.name, allow_no_source=True),
file.data,
count_new=True,
):
actor_name = file.name[
file.name.rindex("/") : file.name.rindex(".")
]
try:
pack_path = util.get_game_file(
f"Actor/Pack/{actor_name}.sbactorpack"
)
stock_data = util.get_nested_file_bytes(
f"{str(pack_path)}//{file.name}"
)
if stock_data:
new_sarc.files[file.name] = stock_data
else:
raise FileNotFoundError(file.name)
except (FileNotFoundError, AttributeError):
error = (
"This mod contains a Havok file not supported by the "
f"converter: {file.name}"
)
else:
error = (
"This mod contains a Havok file not supported by the"
f" converter: {file.name}"
)
else:
if file.data[0:4] == b"AAMP":
continue
try:
hk = Havok.from_bytes(bytes(file.data))
except: # pylint: disable=bare-except
return f"Could not parse Havok file {file.name}"
if to_wiiu:
hk.to_wiiu()
else:
hk.to_switch()
hk.serialize()
new_sarc.files[file.name] = hk.to_bytes()
actor_pack.write_bytes(util.compress(new_sarc.write()[1]))
return error
def _check_modded(file: Path, tmp_dir: Path):
try:
canon = util.get_canon_name(file.relative_to(tmp_dir).as_posix())
except ValueError:
util.vprint(
f"Ignored unknown file {file.relative_to(tmp_dir).as_posix()}")
return None
if util.is_file_modded(canon, file, True):
util.vprint(f"Found modded file {canon}")
return file
else:
if "Aoc/0010/Map/MainField" in canon:
file.unlink()
util.vprint(f"Ignored unmodded file {canon}")
return None
def find_modded_sarc_files(mod_sarc: Union[Path, oead.Sarc],
tmp_dir: Path,
name: str = "",
aoc: bool = False) -> List[str]:
if isinstance(mod_sarc, Path):
if any(mod_sarc.name.startswith(exclude) for exclude in ["Bootup_"]):
return []
name = str(mod_sarc.relative_to(tmp_dir))
aoc = util.get_dlc_path() in mod_sarc.parts or "Aoc" in mod_sarc.parts
try:
mod_sarc = oead.Sarc(util.unyaz_if_needed(mod_sarc.read_bytes()))
except (RuntimeError, ValueError, oead.InvalidDataError):
return []
modded_files = []
for file, contents in [(f.name, bytes(f.data))
for f in mod_sarc.get_files()]:
canon = file.replace(".s", ".")
if aoc:
canon = "Aoc/0010/" + canon
contents = util.unyaz_if_needed(contents)
nest_path = str(name).replace("\\", "/") + "//" + file
if util.is_file_modded(canon, contents, True):
modded_files.append(nest_path)
util.vprint(
f'Found modded file {canon} in {str(name).replace("//", "/")}')
if util.is_file_sarc(canon) and ".ssarc" not in file:
try:
nest_sarc = oead.Sarc(contents)
except ValueError:
continue
sub_mod_files = find_modded_sarc_files(nest_sarc,
name=nest_path,
tmp_dir=tmp_dir,
aoc=aoc)
modded_files.extend(sub_mod_files)
else:
util.vprint(
f'Ignored unmodded file {canon} in {str(name).replace("//", "/")}'
)
return modded_files
def merge_sarcs(file_name: str, sarcs: List[Union[Path,
bytes]]) -> (str, bytes):
opened_sarcs: List[oead.Sarc] = []
if "ThunderRodLv2" in file_name:
print()
if isinstance(sarcs[0], Path):
for i, sarc_path in enumerate(sarcs):
sarcs[i] = sarc_path.read_bytes()
for sarc_bytes in sarcs:
sarc_bytes = util.unyaz_if_needed(sarc_bytes)
try:
opened_sarcs.append(oead.Sarc(sarc_bytes))
except (ValueError, RuntimeError, oead.InvalidDataError):
continue
all_files = {
file.name
for open_sarc in opened_sarcs for file in open_sarc.get_files()
}
nested_sarcs = {}
new_sarc = oead.SarcWriter(endian=oead.Endianness.Big if util.get_settings(
"wiiu") else oead.Endianness.Little)
files_added = set()
for opened_sarc in reversed(opened_sarcs):
for file in [
f for f in opened_sarc.get_files() if f.name not in files_added
]:
file_data = oead.Bytes(file.data)
if (file.name[file.name.rindex("."):] in util.SARC_EXTS -
EXCLUDE_EXTS) and file.name not in SPECIAL:
if file.name not in nested_sarcs:
nested_sarcs[file.name] = []
nested_sarcs[file.name].append(util.unyaz_if_needed(file_data))
elif util.is_file_modded(file.name.replace(".s", "."),
file_data,
count_new=True):
new_sarc.files[file.name] = file_data
files_added.add(file.name)
util.vprint(set(nested_sarcs.keys()))
for file, sarcs in nested_sarcs.items():
if not sarcs:
continue
merged_bytes = merge_sarcs(file, sarcs[::-1])[1]
if Path(file).suffix.startswith(".s") and not file.endswith(".sarc"):
merged_bytes = util.compress(merged_bytes)
new_sarc.files[file] = merged_bytes
files_added.add(file)
for file in [file for file in all_files if file not in files_added]:
for opened_sarc in [
open_sarc for open_sarc in opened_sarcs
if (file in [f.name for f in open_sarc.get_files()])
]:
new_sarc.files[file] = oead.Bytes(opened_sarc.get_file(file).data)
break
if "Bootup.pack" in file_name:
for merger in [
merger() for merger in mergers.get_mergers()
if merger.is_bootup_injector()
]:
inject = merger.get_bootup_injection()
if not inject:
continue
file, data = inject
new_sarc.files[file] = data
return (file_name, bytes(new_sarc.write()[1]))
def find_modded_files(tmp_dir: Path,
verbose: bool = False,
original_pool: Pool = None) -> List[Union[Path, str]]:
"""
Detects all of the modified files in an extracted mod
:param tmp_dir: The path to the base directory of the mod.
:type tmp_dir: class:`pathlib.Path`
:param deep_merge: Whether to log diffs for individual AAMP and BYML files, defaults to False
:type deep_merge: bool, optional
:param verbose: Specifies whether to return more detailed output
:type verbose: bool, optional
:returns: Returns a tuple with a dict of modified files and the RSTB entries, a list of changes,
and (if deep merge) diffs of modded BYML and AAMP files
:rtype: (dict of class:`pathlib.Path`: int, list of str, dict of str: str)
"""
modded_files = []
if isinstance(tmp_dir, str):
tmp_dir = Path(tmp_dir)
rstb_path = tmp_dir / 'content' / 'System' / 'Resource' /\
'ResourceSizeTable.product.srsizetable'
if rstb_path.exists():
rstb_path.unlink()
if (tmp_dir / 'aoc').exists:
try:
util.get_aoc_dir()
except FileNotFoundError as err:
err.error_text = (
'This mod uses DLC files, but you do not appear to have the DLC '
'installed. If you still want to use this mod, unpack it and '
'remove the "aoc" folder.')
raise err
aoc_field = tmp_dir / 'aoc' / '0010' / 'Pack' / 'AocMainField.pack'
if aoc_field.exists() and aoc_field.stat().st_size > 0:
with aoc_field.open('rb') as a_file:
sarc.read_file_and_make_sarc(a_file).extract_to_dir(
str(tmp_dir / 'aoc' / '0010'))
aoc_field.write_bytes(b'')
for file in tmp_dir.rglob('**/*'):
if file.is_file():
canon = util.get_canon_name(file.relative_to(tmp_dir).as_posix())
if canon is None:
if verbose:
print(
f'Ignored unknown file {file.relative_to(tmp_dir).as_posix()}'
)
continue
if util.is_file_modded(canon, file, True):
modded_files.append(file)
if verbose:
print(f'Found modded file {canon}')
else:
if 'Aoc/0010/Map/MainField' in canon:
file.unlink()
if verbose:
print(f'Ignored unmodded file {canon}')
continue
total = len(modded_files)
print(f'Found {total} modified file{"s" if total > 1 else ""}')
total = 0
sarc_files = [
file for file in modded_files if file.suffix in util.SARC_EXTS
]
if sarc_files:
print(f'Scanning files packed in SARCs...')
num_threads = min(len(sarc_files), cpu_count() - 1)
pool = original_pool or Pool(processes=num_threads)
modded_sarc_files = pool.map(
partial(find_modded_sarc_files, tmp_dir=tmp_dir, verbose=verbose),
sarc_files)
for files in modded_sarc_files:
total += len(files)
modded_files.extend(files)
if not original_pool:
pool.close()
pool.join()
print(f'Found {total} modified packed file{"s" if total > 1 else ""}')
return modded_files
def find_modded_sarc_files(mod_sarc: Union[Path, sarc.SARC],
tmp_dir: Path,
name: str = '',
aoc: bool = False,
verbose: bool = False) -> List[str]:
"""
Detects all of the modified files in a SARC
:param mod_sarc: The SARC to scan for modded files.
:type mod_sarc: class:`sarc.SARC`
:param tmp_dir: The path to the base directory of the mod.
:type tmp_dir: class:`pathlib.Path`
:param name: The name of the SARC which contains the current SARC.
:type name: str
:param aoc: Specifies whether the SARC is DLC content, defaults to False.
:type aoc: bool, optional
:param nest_level: The depth to which the current SARC is nested in more SARCs, defaults to 0
:type nest_level: int, optional
:param deep_merge: Whether to log diffs for individual AAMP and BYML files, defaults to False
:type deep_merge: bool, optional
:param verbose: Specifies whether to return more detailed output
:type verbose: bool, optional
"""
if isinstance(mod_sarc, Path):
if any(mod_sarc.name.startswith(exclude) for exclude in ['Bootup_']):
return []
name = str(mod_sarc.relative_to(tmp_dir))
aoc = 'aoc' in mod_sarc.parts or 'Aoc' in mod_sarc.parts
with mod_sarc.open('rb') as s_file:
mod_sarc = sarc.read_file_and_make_sarc(s_file)
if not mod_sarc:
return []
modded_files = []
for file in mod_sarc.list_files():
canon = file.replace('.s', '.')
if aoc:
canon = 'Aoc/0010/' + canon
contents = mod_sarc.get_file_data(file).tobytes()
contents = util.unyaz_if_needed(contents)
nest_path = str(name).replace('\\', '/') + '//' + file
if util.is_file_modded(canon, contents, True):
modded_files.append(nest_path)
if verbose:
print(
f'Found modded file {canon} in {str(name).replace("//", "/")}'
)
if util.is_file_sarc(canon) and '.ssarc' not in file:
try:
nest_sarc = sarc.SARC(contents)
except ValueError:
continue
sub_mod_files = find_modded_sarc_files(nest_sarc,
name=nest_path,
tmp_dir=tmp_dir,
aoc=aoc,
verbose=verbose)
modded_files.extend(sub_mod_files)
else:
if verbose:
print(
f'Ignored unmodded file {canon} in {str(name).replace("//", "/")}'
)
return modded_files
