def bootup_from_msbts(
lang: str = 'USen',
msbt_dir: Path = util.get_work_dir() / 'tmp_text' / 'merged'
) -> (Path, int):
"""
Generates a new Bootup_XXxx.pack from a directory of MSBT files
:param lang: The game language to use, defaults to USen.
:type lang: str, optional
:param msbt_dir: The directory to pull MSBTs from, defaults to "tmp_text/merged" in BCML's
working directory.
:type msbt_dir: class:`pathlib.Path`, optional
:returns: A tuple with the path to the new Bootup_XXxx.pack and the RSTB size of the new
Msg_XXxx.product.sarc
:rtype: (class:`pathlib.Path`, int)
"""
new_boot_path = msbt_dir.parent / f'Bootup_{lang}.pack'
with new_boot_path.open('wb') as new_boot:
s_msg = sarc.SARCWriter(True)
for new_msbt in msbt_dir.rglob('**/*.msbt'):
with new_msbt.open('rb') as f_new:
s_msg.add_file(
str(new_msbt.relative_to(msbt_dir)).replace('\\', '/'),
f_new.read())
new_msg_stream = io.BytesIO()
s_msg.write(new_msg_stream)
unyaz_bytes = new_msg_stream.getvalue()
rsize = rstb.SizeCalculator().calculate_file_size_with_ext(
unyaz_bytes, True, '.sarc')
new_msg_bytes = util.compress(unyaz_bytes)
s_boot = sarc.SARCWriter(True)
s_boot.add_file(f'Message/Msg_{lang}.product.ssarc', new_msg_bytes)
s_boot.write(new_boot)
return new_boot_path, rsize
def get_modded_texts(modded_msyts: list, lang: str = 'USen', tmp_dir: Path = \
util.get_work_dir() / 'tmp_text') -> dict:
"""
Builds a dictionary of all edited text entries in modded MSYTs
:param modded_msyts: A list of MSYT files that have been modified.
:type modded_msyts: list of str
:param tmp_dir: The temp directory to use, defaults to "tmp_text" in BCML's working directory.
:type tmp_dir: class:`pathlib.Path`
:returns: Returns a dictionary of modified MSYT text entries.
:rtype: dict
"""
text_edits = {}
check_msyts = [
msyt for msyt in list(tmp_dir.rglob('**/*.msyt'))
if str(msyt.relative_to(tmp_dir)).replace('\\', '/') in modded_msyts
]
if not check_msyts:
return {}
num_threads = min(multiprocessing.cpu_count(), len(check_msyts))
thread_checker = partial(threaded_compare_texts,
tmp_dir=tmp_dir,
lang=lang)
pool = multiprocessing.Pool(processes=num_threads)
edit_results = pool.map(thread_checker, check_msyts)
pool.close()
pool.join()
for edit in edit_results:
rel_path, edits = edit
if edits is None:
print(f'{rel_path} is corrupt and will not be merged.')
continue
if edits['entries']:
text_edits[rel_path] = edits
return text_edits
def msbt_to_msyt(tmp_dir: Path = util.get_work_dir() / 'tmp_text'):
""" Converts MSBTs in given temp dir to MSYTs """
subprocess.run([
str(util.get_exec_dir() / 'helpers' / 'msyt.exe'), 'export', '-d',
str(tmp_dir)
],
creationflags=util.CREATE_NO_WINDOW)
fix_msbts = [
msbt for msbt in tmp_dir.rglob('**/*.msbt')
if not msbt.with_suffix('.msyt').exists()
]
if fix_msbts:
print('Some MSBTs failed to convert. Trying again individually...')
pool = multiprocessing.Pool(
processes=min(multiprocessing.cpu_count(), len(fix_msbts)))
pool.map(_msyt_file, fix_msbts)
pool.close()
pool.join()
fix_msbts = [
msbt for msbt in tmp_dir.rglob('**/*.msbt')
if not msbt.with_suffix('.msyt').exists()
]
if fix_msbts:
print(
f'{len(fix_msbts)} MSBT files failed to convert. They will not be merged.'
)
for msbt_file in tmp_dir.rglob('**/*.msbt'):
Path(msbt_file).unlink()
return fix_msbts
def extract_ref_msyts(lang: str = 'USen',
for_merge: bool = False,
tmp_dir: Path = util.get_work_dir() / 'tmp_text'):
"""
Extracts the reference MSYT texts for the given language to a temp dir
:param lang: The game language to use, defaults to USen.
:type lang: str, optional
:param for_merge: Whether the output is to be merged (or as reference), defaults to False
:type for_merge: bool
:param tmp_dir: The temp directory to extract to, defaults to "tmp_text" in BCML's working
directory.
:type tmp_dir: class:`pathlib.Path`, optional
"""
if tmp_dir.exists():
shutil.rmtree(tmp_dir, ignore_errors=True)
with util.get_game_file(f'Pack/Bootup_{lang}.pack').open('rb') as b_file:
bootup_pack = sarc.read_file_and_make_sarc(b_file)
msg_bytes = util.decompress(
bootup_pack.get_file_data(
f'Message/Msg_{lang}.product.ssarc').tobytes())
msg_pack = sarc.SARC(msg_bytes)
if not for_merge:
merge_dir = tmp_dir / 'ref'
else:
merge_dir = tmp_dir / 'merged'
msg_pack.extract_to_dir(str(merge_dir))
msbt_to_msyt(merge_dir)
def get_modded_msyts(msg_sarc: sarc.SARC, lang: str = 'USen',
tmp_dir: Path = util.get_work_dir() / 'tmp_text') -> (list, dict):
"""
Gets a list of modified game text files in a given message SARC
:param msg_sarc: The message SARC to scan for changes.
:type msg_sarc: class:`sarc.SARC`
:param lang: The game language to use, defaults to USen.
:type lang: str, optional
:param tmp_dir: The temp directory to use, defaults to "tmp_text" in BCML's working directory.
:type tmp_dir: class:`pathlib.Path`, optional
:returns: Returns a tuple containing a list of modded text files and a dict of new text
files with their contents.
:rtype: (list of str, dict of str: bytes)
"""
hashes = get_msbt_hashes(lang)
modded_msyts = []
added_msbts = {}
write_msbts = []
for msbt in msg_sarc.list_files():
if any(exclusion in msbt for exclusion in EXCLUDE_TEXTS):
continue
m_data = msg_sarc.get_file_data(msbt)
m_hash = xxhash.xxh32(m_data).hexdigest()
if msbt not in hashes:
added_msbts[msbt] = m_data
elif m_hash != hashes[msbt]:
write_msbts.append((tmp_dir / msbt, m_data.tobytes()))
modded_msyts.append(msbt.replace('.msbt', '.msyt'))
if write_msbts:
pool = multiprocessing.Pool()
pool.map(write_msbt, write_msbts)
pool.close()
pool.join()
return modded_msyts, added_msbts
def get_text_mods_from_bootup(bootup_path: Union[Path, str],
tmp_dir: Path = util.get_work_dir() / 'tmp_text',
verbose: bool = False, lang: str = ''):
"""
Detects modifications to text files inside a given Bootup_XXxx.pack
:param bootup_path: Path to the Bootup_XXxx.pack file.
:type bootup_path: class:`pathlib.Path`
:param tmp_dir: The temp directory to use, defaults to "tmp_text" in BCML's working directory.
:type tmp_dir: class:`pathlib.Path`
:param verbose: Whether to display more detailed output, defaults to False.
:type verbose: bool, optional
:returns: Return a tuple containing a dict of modded text entries, a SARC containing added text
MSBTs, and the game language of the bootup pack.
:rtype: (dict, class:`sarc.SARCWriter`, str)
"""
if not lang:
lang = util.get_file_language(bootup_path)
print(f'Scanning text modifications for language {lang}...')
spaces = ' '
if verbose:
print(f'{spaces}Identifying modified text files...')
with open(bootup_path, 'rb') as b_file:
bootup_sarc = sarc.read_file_and_make_sarc(b_file)
msg_bytes = util.decompress(bootup_sarc.get_file_data(f'Message/Msg_{lang}.product.ssarc'))
msg_sarc = sarc.SARC(msg_bytes)
if not msg_sarc:
print(f'Failed to open Msg_{lang}.product.ssarc, could not analyze texts')
modded_msyts, added_msbts = get_modded_msyts(msg_sarc, lang)
added_text_store = None
if added_msbts:
added_text_store = store_added_texts(added_msbts)
if verbose:
for modded_text in modded_msyts:
print(f'{spaces}{spaces}{modded_text} has been changed')
for added_text in added_msbts:
print(f'{spaces}{spaces}{added_text} has been added')
problems = msbt_to_msyt()
for problem in problems:
msyt_name = problem.relative_to(tmp_dir).with_suffix('.msyt').as_posix()
try:
modded_msyts.remove(msyt_name)
except ValueError:
pass
if verbose:
print(f'{spaces}Scanning texts files for modified entries...')
modded_texts = get_modded_texts(modded_msyts, lang=lang)
s_modded = 's' if len(modded_texts) != 1 else ''
s_added = 's' if len(added_msbts) != 1 else ''
print(f'Language {lang} has total {len(modded_texts)} modified text file{s_modded} and '
f'{len(added_msbts)} new text file{s_added}')
shutil.rmtree(tmp_dir)
return modded_texts, added_text_store, lang
def msyt_to_msbt(tmp_dir: Path = util.get_work_dir() / 'tmp_text'):
""" Converts merged MSYTs in given temp dir to MSBTs """
msyt_bin = util.get_exec_dir() / 'helpers' / 'msyt.exe'
merge_dir = tmp_dir / 'merged'
m_args = [str(msyt_bin), 'create', '-d', str(merge_dir),
'-p', 'wiiu', '-o', str(merge_dir)]
subprocess.run(m_args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, creationflags=util.CREATE_NO_WINDOW)
for merged_msyt in merge_dir.rglob('**/*.msyt'):
merged_msyt.unlink()
def merge_texts(lang: str = 'USen',
tmp_dir: Path = util.get_work_dir() / 'tmp_text',
verbose: bool = False,
original_pool: multiprocessing.Pool = None):
"""
Merges installed text mods and saves the new Bootup_XXxx.pack, fixing the RSTB if needed
:param lang: The game language to use, defaults to USen.
:type lang: str, optional
:param tmp_dir: The temp directory to extract to, defaults to "tmp_text" in BCML's work dir.
:type tmp_dir: class:`pathlib.Path`, optional
:param verbose: Whether to display more detailed output, defaults to False
:type verbose: bool, optional
"""
print(f'Loading text mods for language {lang}...')
text_mods = get_modded_text_entries(lang)
if not text_mods:
print('No text merging necessary.')
old_path = util.get_master_modpack_dir() / 'content' / 'Pack' / \
f'Bootup_{lang}.pack'
if old_path.exists():
old_path.unlink()
return
if verbose:
print(f' Found {len(text_mods)} text mods to be merged')
if tmp_dir.exists():
if verbose:
print('Cleaning temp directory...')
shutil.rmtree(tmp_dir, ignore_errors=True)
print('Extracting clean MSYTs...')
try:
extract_ref_msyts(lang, for_merge=True, tmp_dir=tmp_dir)
except FileNotFoundError:
return
merge_dir = tmp_dir / 'merged'
merge_dir.mkdir(parents=True, exist_ok=True)
print('Merging modified text files...')
modded_text_files = list(merge_dir.rglob('**/*.msyt'))
num_threads = min(multiprocessing.cpu_count(), len(modded_text_files))
pool = original_pool or multiprocessing.Pool(processes=num_threads)
thread_merger = partial(threaded_merge_texts,
merge_dir=merge_dir,
text_mods=text_mods,
verbose=verbose)
results = pool.map(thread_merger, modded_text_files)
for merge_count, rel_path in results:
if merge_count > 0:
print(f' Merged {merge_count} versions of {rel_path}')
if not original_pool:
pool.close()
pool.join()
print('Generating merged MSBTs...')
msyt_to_msbt(tmp_dir)
added_texts = get_added_text_mods(lang)
if added_texts:
print('Adding mod-original MSBTs...')
for added_text in added_texts:
for msbt in added_text.list_files():
Path(merge_dir / msbt).parent.mkdir(parents=True,
exist_ok=True)
Path(merge_dir / msbt).write_bytes(
added_text.get_file_data(msbt).tobytes())
print(f'Creating new Bootup_{lang}.pack...')
tmp_boot_path = bootup_from_msbts(lang)[0]
merged_boot_path = util.get_modpack_dir() / '9999_BCML' / 'content' / \
'Pack' / f'Bootup_{lang}.pack'
if merged_boot_path.exists():
if verbose:
print(f' Removing old Bootup_{lang}.pack...')
merged_boot_path.unlink()
merged_boot_path.parent.mkdir(parents=True, exist_ok=True)
shutil.copy(str(tmp_boot_path), str(merged_boot_path))
rstb_path = util.get_modpack_dir() / '9999_BCML' / 'content' / 'System' / 'Resource' /\
'ResourceSizeTable.product.srsizetable'
if rstb_path.exists():
table: rstb.ResourceSizeTable = rstb.util.read_rstb(
str(rstb_path), True)
else:
table = rstable.get_stock_rstb()
msg_path = f'Message/Msg_{lang}.product.sarc'
if table.is_in_table(msg_path):
print('Correcting RSTB...')
table.delete_entry(msg_path)
rstb_path.parent.mkdir(parents=True, exist_ok=True)
rstb.util.write_rstb(table, str(rstb_path), True)