def _get_sizes_in_sarc(
file: Union[Path, oead.Sarc], guess: bool, is_aoc: bool = False
) -> {}:
sizes = {}
if isinstance(file, Path):
is_aoc = util.get_dlc_path() in file.as_posix()
try:
file = oead.Sarc(util.unyaz_if_needed(file.read_bytes()))
except (RuntimeError, oead.InvalidDataError):
print(f"{file} could not be opened")
return {}
for nest_file, data in [(file.name, file.data) for file in file.get_files()]:
canon = nest_file.replace(".s", ".")
if data[0:4] == b"Yaz0":
data = util.decompress(data)
ext = Path(canon).suffix
if (
util.is_file_modded(canon, data)
and ext not in EXCLUDE_EXTS
and canon not in EXCLUDE_NAMES
):
sizes[canon] = calculate_size(canon, data, guess=guess)
if ext in util.SARC_EXTS - SARC_EXCLUDES:
try:
nest_sarc = oead.Sarc(data)
except (ValueError, RuntimeError, oead.InvalidDataError):
continue
sizes.update(_get_sizes_in_sarc(nest_sarc, guess, is_aoc=is_aoc))
del nest_sarc
del data
del file
return sizes
def __init__(self) -> None:
self._gameid = "5866" if util.get_settings("wiiu") else "6386"
if not GB_DATA.exists():
GB_DATA.write_bytes(
util.decompress(
(util.get_exec_dir() / "data" / "gb.sjson").read_bytes()))
self._data = json.loads(GB_DATA.read_text("utf-8"))
def generate_diff(self, mod_dir: Path, modded_files: List[Union[Path,
str]]):
if (f"{util.get_content_path()}/Pack/TitleBG.pack//Quest/QuestProduct.sbquestpack"
not in modded_files):
return {}
print("Logging modified quests...")
stock_quests = get_stock_quests()
stock_names = [q["Name"] for q in stock_quests]
title_sarc = oead.Sarc((mod_dir / util.get_content_path() / "Pack" /
"TitleBG.pack").read_bytes())
mod_quests = oead.byml.from_binary(
util.decompress(
title_sarc.get_file("Quest/QuestProduct.sbquestpack").data))
mod_names = [q["Name"] for q in mod_quests]
diffs = oead.byml.Hash({
"add":
oead.byml.Array(),
"mod":
oead.byml.Hash(),
"del":
oead.byml.Array({q
for q in stock_names if q not in mod_names}),
})
for i, quest in enumerate(mod_quests):
quest_name = quest["Name"]
quest["prev_quest"] = mod_quests[
i - 1]["Name"] if i > 0 else "--index_zero"
if quest_name not in stock_names:
diffs["add"].append(quest)
elif quest != stock_quests[stock_names.index(quest_name)]:
diffs["mod"][quest_name] = quest
return diffs
def get_text_hashes(language: str = None) -> {}:
hashes = json.loads(
util.decompress((util.get_exec_dir() / "data" / "hashes" /
"msyts.sjson").read_bytes()).decode("utf8"))
if language:
return hashes[language if not language.endswith("en") else "XXen"]
else:
return hashes
def generate_diff(self, mod_dir: Path, modded_files: List[Union[Path, str]]):
if 'content/Pack/Bootup.pack//Event/EventInfo.product.sbyml' in modded_files:
with (mod_dir / 'content' / 'Pack' / 'Bootup.pack').open('rb') as bootup_file:
bootup_sarc = sarc.read_file_and_make_sarc(bootup_file)
event_info = byml.Byml(
util.decompress(
bootup_sarc.get_file_data('Event/EventInfo.product.sbyml').tobytes()
)
).parse()
return get_modded_events(event_info)
else:
return {}
def threaded_merge(item, verbose: bool) -> (str, dict):
"""Deep merges an individual file, suitable for multiprocessing"""
file, stuff = item
failures = {}
base_file = util.get_game_file(file, file.startswith('aoc'))
if (util.get_master_modpack_dir() / file).exists():
base_file = util.get_master_modpack_dir() / file
file_ext = os.path.splitext(file)[1]
if file_ext in util.SARC_EXTS and (util.get_master_modpack_dir() /
file).exists():
base_file = (util.get_master_modpack_dir() / file)
file_bytes = base_file.read_bytes()
yazd = file_bytes[0:4] == b'Yaz0'
file_bytes = file_bytes if not yazd else util.decompress(file_bytes)
magic = file_bytes[0:4]
if magic == b'SARC':
new_sarc, sub_failures = nested_patch(sarc.SARC(file_bytes), stuff)
del file_bytes
new_bytes = new_sarc.get_bytes()
for failure, contents in sub_failures.items():
print(f'Some patches to {failure} failed to apply.')
failures[failure] = contents
else:
try:
if magic == b'AAMP':
aamp_contents = aamp.Reader(file_bytes).parse()
for change in stuff:
aamp_contents = _aamp_merge(aamp_contents, change)
aamp_bytes = aamp.Writer(aamp_contents).get_bytes()
del aamp_contents
new_bytes = aamp_bytes if not yazd else util.compress(
aamp_bytes)
else:
raise ValueError(f'{file} is not a SARC or AAMP file.')
except ValueError:
new_bytes = file_bytes
del file_bytes
print(f'Deep merging file {file} failed. No changes were made.')
new_bytes = new_bytes if not yazd else util.compress(new_bytes)
output_file = (util.get_master_modpack_dir() / file)
if base_file == output_file:
output_file.unlink()
output_file.parent.mkdir(parents=True, exist_ok=True)
output_file.write_bytes(new_bytes)
del new_bytes
if magic == b'SARC' and verbose:
print(f'Finished patching files inside {file}')
elif verbose:
print(f'Finished patching {file}')
return util.get_canon_name(file), failures
def nested_patch(pack: sarc.SARC, nest: dict) -> (sarc.SARCWriter, dict):
"""
Recursively patches deep merge files in a SARC
:param pack: The SARC in which to recursively patch.
:type pack: class:`sarc.SARC`
:param nest: A dict of nested patches to apply.
:type nest: dict
:return: Returns a new SARC with patches applied and a dict of any failed patches.
:rtype: (class:`sarc.SARCWriter`, dict, dict)
"""
new_sarc = sarc.make_writer_from_sarc(pack)
failures = {}
for file, stuff in nest.items():
file_bytes = pack.get_file_data(file).tobytes()
yazd = file_bytes[0:4] == b'Yaz0'
file_bytes = util.decompress(file_bytes) if yazd else file_bytes
if isinstance(stuff, dict):
sub_sarc = sarc.SARC(file_bytes)
new_sarc.delete_file(file)
new_sub_sarc, sub_failures = nested_patch(sub_sarc, stuff)
for failure in sub_failures:
failure[file + '//' + failure] = sub_failures[failure]
del sub_sarc
new_bytes = new_sub_sarc.get_bytes()
new_sarc.add_file(
file, new_bytes if not yazd else util.compress(new_bytes))
elif isinstance(stuff, list):
try:
if file_bytes[0:4] == b'AAMP':
aamp_contents = aamp.Reader(file_bytes).parse()
for change in stuff:
aamp_contents = _aamp_merge(aamp_contents, change)
aamp_bytes = aamp.Writer(aamp_contents).get_bytes()
del aamp_contents
new_bytes = aamp_bytes if not yazd else util.compress(
aamp_bytes)
cache_merged_aamp(file, new_bytes)
else:
raise ValueError(
'Wait, what the heck, this isn\'t an AAMP file?!')
except ValueError:
new_bytes = pack.get_file_data(file).tobytes()
print(f'Deep merging {file} failed. No changed were made.')
new_sarc.delete_file(file)
new_sarc.add_file(file, new_bytes)
return new_sarc, failures
def nested_patch(pack: oead.Sarc, nest: dict) -> Tuple[oead.SarcWriter, dict]:
new_sarc: oead.SarcWriter = oead.SarcWriter.from_sarc(pack)
failures: dict = {}
for file, stuff in nest.items():
file_bytes = pack.get_file(file).data
yazd = file_bytes[0:4] == b"Yaz0"
file_bytes = util.decompress(file_bytes) if yazd else file_bytes
if isinstance(stuff, dict):
sub_sarc = oead.Sarc(file_bytes)
new_sub_sarc, sub_failures = nested_patch(sub_sarc, stuff)
for failure in sub_failures:
failure[file + "//" + failure] = sub_failures[failure]
del sub_sarc
new_bytes = bytes(new_sub_sarc.write()[1])
new_sarc.files[file] = new_bytes if not yazd else util.compress(
new_bytes)
elif isinstance(stuff, ParameterList):
try:
if file_bytes[0:4] == b"AAMP":
aamp_contents = ParameterIO.from_binary(file_bytes)
try:
file_ext = os.path.splitext(file)[1]
aamp_contents = shop_merge(
aamp_contents,
file_ext.replace(".", ""),
stuff.lists["Additions"],
stuff.lists["Removals"],
)
aamp_bytes = ParameterIO.to_binary(aamp_contents)
except: # pylint: disable=bare-except
raise RuntimeError(
f"AAMP file {file} could be merged.")
del aamp_contents
new_bytes = aamp_bytes if not yazd else util.compress(
aamp_bytes)
cache_merged_shop(file, new_bytes)
else:
raise ValueError(
"Wait, what the heck, this isn't an AAMP file?!")
except ValueError:
new_bytes = pack.get_file(file).data
print(f"Deep merging {file} failed. No changes were made.")
new_sarc.files[file] = oead.Bytes(new_bytes)
return new_sarc, failures
def generate_diff(self, mod_dir: Path, modded_files: List[Union[Path,
str]]):
if (f"{util.get_content_path()}/Pack/Bootup.pack//GameData/gamedata.ssarc"
in modded_files):
print("Logging changes to game data flags...")
bootup_sarc = oead.Sarc(
util.unyaz_if_needed((mod_dir / util.get_content_path() /
"Pack" / "Bootup.pack").read_bytes()))
data_sarc = oead.Sarc(
util.decompress(
bootup_sarc.get_file("GameData/gamedata.ssarc").data))
diff = get_modded_gamedata_entries(data_sarc, pool=self._pool)
del bootup_sarc
del data_sarc
return diff
else:
return {}
def generate_diff(self, mod_dir: Path, modded_files: List[Union[Path,
str]]):
try:
actor_file = next(
iter([
file for file in modded_files
if Path(file).name == "ActorInfo.product.sbyml"
]))
except StopIteration:
return {}
print("Detecting modified actor info entries...")
data = util.decompress(actor_file.read_bytes())
try:
actorinfo = oead.byml.from_binary(data)
except oead.InvalidDataError:
data = bytearray(data)
data[3] = 2
try:
actorinfo = oead.byml.from_binary(data)
except oead.InvalidDataError as err:
raise ValueError(
"This mod contains a corrupt actor info file.") from err
del actor_file
stock_actorinfo = get_stock_actorinfo()
stock_actors = {
actor["name"]: actor
for actor in stock_actorinfo["Actors"]
}
del stock_actorinfo
diff = oead.byml.Hash({
**{
str(crc32(actor["name"].encode("utf8"))): actor
for actor in actorinfo["Actors"] if actor["name"] not in stock_actors
},
**{
str(crc32(actor["name"].encode("utf8"))): {
key: value
for key, value in actor.items() if (key not in stock_actors[actor["name"]] or value != stock_actors[actor["name"]][key])
}
for actor in actorinfo["Actors"] if actor["name"] in stock_actors and actor != stock_actors[actor["name"]]
},
})
del actorinfo
del stock_actors
return diff
def inject_files_into_bootup(files: list, datas: list):
bootup_path: Path = root_dir() / "content" / "Pack" / "Bootup.pack"
sarc_data = bootup_path.read_bytes()
yaz = sarc_data[0:4] == b"Yaz0"
if yaz:
sarc_data = bcmlutil.decompress(sarc_data)
old_sarc = oead.Sarc(sarc_data)
del sarc_data
new_sarc = oead.SarcWriter.from_sarc(old_sarc)
del old_sarc
for idx in range(len(files)):
new_sarc.files[files[idx]] = (datas[idx] if isinstance(
datas[idx], bytes) else bytes(datas[idx]))
new_bytes = new_sarc.write()[1]
del new_sarc
bootup_path.write_bytes(
new_bytes if not yaz else bcmlutil.compress(new_bytes))
del new_bytes
def generate_diff(self, mod_dir: Path, modded_files: List[Union[str,
Path]]):
needle = f"{util.get_content_path()}/Pack/Bootup.pack//Ecosystem/StatusEffectList.sbyml"
if needle not in modded_files:
return {}
print("Logging changes to effect status levels...")
stock_effects = get_stock_effects()
bootup_sarc = oead.Sarc((mod_dir / util.get_content_path() / "Pack" /
"Bootup.pack").read_bytes())
mod_effects = oead.byml.from_binary(
util.decompress(
bootup_sarc.get_file(
"Ecosystem/StatusEffectList.sbyml").data))[0]
diff = oead.byml.Hash({
effect: params
for effect, params in mod_effects.items()
if stock_effects[effect] != params
})
del stock_effects
del mod_effects
def generate_diff(self, mod_dir: Path, modded_files: List[Union[Path, str]]):
if (
f"{util.get_content_path()}/Pack/Bootup.pack//Ecosystem/AreaData.sbyml"
in modded_files
):
print("Logging modded areadata...")
bootup_sarc = oead.Sarc(
(
mod_dir / util.get_content_path() / "Pack" / "Bootup.pack"
).read_bytes()
)
areadata = oead.byml.from_binary(
util.decompress(bootup_sarc.get_file("Ecosystem/AreaData.sbyml").data)
)
diff = get_modded_areadata(areadata)
del bootup_sarc
del areadata
return diff
else:
return {}
def generate_diff(self, mod_dir: Path, modded_files: List[Union[Path, str]]):
if (
f"{util.get_content_path()}/Pack/Bootup.pack//Event/EventInfo.product.sbyml"
in modded_files
):
print("Logging modded events...")
bootup_sarc = oead.Sarc(
(mod_dir / util.get_content_path() / "Pack" / "Bootup.pack").read_bytes()
)
event_info = oead.byml.from_binary(
util.decompress(
bootup_sarc.get_file("Event/EventInfo.product.sbyml").data
)
)
diff = get_modded_events(event_info)
del bootup_sarc
del event_info
return diff
else:
return {}
def diff_language(bootup: Path, pool: multiprocessing.Pool = None) -> {}:
diff = {}
language = bootup.name[7:-5]
bootup_sarc = oead.Sarc(bootup.read_bytes())
msg_sarc = oead.Sarc(
util.decompress(
bootup_sarc.get_file(
f"Message/Msg_{language}.product.ssarc").data))
with TemporaryDirectory() as tmp:
tmp_dir = Path(tmp)
mod_out = tmp_dir / "mod"
print("Extracting mod texts...")
for file in msg_sarc.get_files():
out = mod_out / file.name
out.parent.mkdir(parents=True, exist_ok=True)
out.write_bytes(file.data)
del msg_sarc
print("Converting texts to MSYT...")
msbt_to_msyt(mod_out, pool=pool)
hashes = get_text_hashes(language)
ref_lang = "XXen" if language.endswith("en") else language
print("Extracting reference texts...")
extract_refs(ref_lang, tmp_dir)
ref_dir = tmp_dir / "refs" / ref_lang
this_pool = pool or multiprocessing.Pool(maxtasksperchild=500)
print("Identifying modified text files...")
results = this_pool.map(
partial(diff_msyt, ref_dir=ref_dir, hashes=hashes,
mod_out=mod_out),
mod_out.rglob("**/*.msyt"),
)
if not pool:
this_pool.close()
this_pool.join()
for result in results:
diff.update(result)
return diff
def get_stock_savedata() -> oead.Sarc:
bootup = oead.Sarc(util.get_game_file("Pack/Bootup.pack").read_bytes())
return oead.Sarc(
util.decompress(bootup.get_file("GameData/savedataformat.ssarc").data))
def threaded_merge(item) -> Tuple[str, dict]:
"""Deep merges an individual file, suitable for multiprocessing"""
file, stuff = item
failures = {}
try:
base_file = util.get_game_file(file,
file.startswith(util.get_dlc_path()))
except FileNotFoundError:
return "", {}
if (util.get_master_modpack_dir() / file).exists():
base_file = util.get_master_modpack_dir() / file
file_ext = os.path.splitext(file)[1]
if file_ext in util.SARC_EXTS and (util.get_master_modpack_dir() /
file).exists():
base_file = util.get_master_modpack_dir() / file
file_bytes = base_file.read_bytes()
yazd = file_bytes[0:4] == b"Yaz0"
file_bytes = file_bytes if not yazd else util.decompress(file_bytes)
magic = file_bytes[0:4]
if magic == b"SARC":
new_sarc, sub_failures = nested_patch(oead.Sarc(file_bytes), stuff)
del file_bytes
new_bytes = bytes(new_sarc.write()[1])
for failure, contents in sub_failures.items():
print(f"Some patches to {failure} failed to apply.")
failures[failure] = contents
elif magic == b"AAMP":
try:
aamp_contents = ParameterIO.from_binary(file_bytes)
try:
aamp_contents = shop_merge(
aamp_contents,
file_ext.replace(".", ""),
stuff.lists["Additions"],
stuff.lists["Removals"],
)
aamp_bytes = ParameterIO.to_binary(aamp_contents)
except: # pylint: disable=bare-except
raise RuntimeError(f"AAMP file {file} could be merged.")
del aamp_contents
new_bytes = aamp_bytes if not yazd else util.compress(aamp_bytes)
except ValueError:
new_bytes = file_bytes
del file_bytes
print(f"Deep merging file {file} failed. No changes were made.")
else:
raise ValueError(f"{file} is not a SARC or AAMP file.")
new_bytes = new_bytes if not yazd else util.compress(new_bytes)
output_file = util.get_master_modpack_dir() / file
if base_file == output_file:
output_file.unlink()
output_file.parent.mkdir(parents=True, exist_ok=True)
output_file.write_bytes(new_bytes)
del new_bytes
if magic == b"SARC":
util.vprint(f"Finished patching files inside {file}")
else:
util.vprint(f"Finished patching {file}")
return util.get_canon_name(file), failures
def get_stock_actorinfo() -> oead.byml.Hash:
actorinfo = util.get_game_file("Actor/ActorInfo.product.sbyml")
return oead.byml.from_binary(util.decompress(actorinfo.read_bytes()))
def get_stock_quests() -> oead.byml.Array:
title_sarc = oead.Sarc(
util.get_game_file("Pack/TitleBG.pack").read_bytes())
return oead.byml.from_binary(
util.decompress(
title_sarc.get_file("Quest/QuestProduct.sbquestpack").data))
def get_stock_effects() -> oead.byml.Hash:
bootup_sarc = oead.Sarc(util.get_game_file("Pack/Bootup.pack").read_bytes())
return oead.byml.from_binary(
util.decompress(bootup_sarc.get_file("Ecosystem/StatusEffectList.sbyml").data)
)[0]
