def log_merged_files_rstb():
""" Generates an RSTB log for the master BCML modpack containing merged files """
print('Updating RSTB for merged files...')
diffs = {}
files = [item for item in util.get_master_modpack_dir().rglob('**/*') if item.is_file()]
guess = util.get_settings_bool('guess_merge')
for file in files:
if file.parent == 'logs':
continue
if file.suffix not in RSTB_EXCLUDE_EXTS and file.name not in RSTB_EXCLUDE_NAMES:
size = calculate_size(file)
if size == 0 and guess:
if file.suffix in util.AAMP_EXTS:
size = guess_aamp_size(file)
elif file.suffix in ['.bfres', '.sbfres']:
size = guess_bfres_size(file)
canon = util.get_canon_name(file.relative_to(util.get_master_modpack_dir()))
if canon:
diffs[canon] = size
sarc_files = [file for file in files if util.is_file_sarc(str(file)) \
and file.suffix != '.ssarc']
if sarc_files:
num_threads = min(multiprocessing.cpu_count(), len(sarc_files))
pool = multiprocessing.Pool(processes=num_threads)
results = pool.map(_get_sizes_in_sarc, sarc_files)
pool.close()
pool.join()
for result in results:
diffs.update(result)
with (util.get_master_modpack_dir() / 'logs' / 'rstb.log').open('w', encoding='utf-8') as log:
log.write('name,size,path\n')
for canon, size in diffs.items():
log.write(f'{canon},{size},//\n')
def _get_sizes_in_sarc(file: Union[Path, sarc.SARC]) -> {}:
calc = rstb.SizeCalculator()
sizes = {}
guess = util.get_settings_bool('guess_merge')
if isinstance(file, Path):
with file.open('rb') as s_file:
file = sarc.read_file_and_make_sarc(s_file)
if not file:
return {}
for nest_file in file.list_files():
canon = nest_file.replace('.s', '.')
data = util.unyaz_if_needed(file.get_file_data(nest_file).tobytes())
ext = Path(canon).suffix
if util.is_file_modded(canon, data) and ext not in RSTB_EXCLUDE_EXTS and canon not in RSTB_EXCLUDE_NAMES:
size = calc.calculate_file_size_with_ext(
data,
wiiu=True,
ext=ext
)
if ext == '.bdmgparam':
size = 0
if size == 0 and guess:
if ext in util.AAMP_EXTS:
size = guess_aamp_size(data, ext)
elif ext in ['.bfres', '.sbfres']:
size = guess_bfres_size(data, canon)
sizes[canon] = size
if util.is_file_sarc(nest_file) and not nest_file.endswith('.ssarc'):
try:
nest_sarc = sarc.SARC(data)
except ValueError:
continue
sizes.update(_get_sizes_in_sarc(nest_sarc))
return sizes
def find_modded_sarc_files(mod_sarc: Union[Path, oead.Sarc],
tmp_dir: Path,
name: str = "",
aoc: bool = False) -> List[str]:
if isinstance(mod_sarc, Path):
if any(mod_sarc.name.startswith(exclude) for exclude in ["Bootup_"]):
return []
name = str(mod_sarc.relative_to(tmp_dir))
aoc = util.get_dlc_path() in mod_sarc.parts or "Aoc" in mod_sarc.parts
try:
mod_sarc = oead.Sarc(util.unyaz_if_needed(mod_sarc.read_bytes()))
except (RuntimeError, ValueError, oead.InvalidDataError):
return []
modded_files = []
for file, contents in [(f.name, bytes(f.data))
for f in mod_sarc.get_files()]:
canon = file.replace(".s", ".")
if aoc:
canon = "Aoc/0010/" + canon
contents = util.unyaz_if_needed(contents)
nest_path = str(name).replace("\\", "/") + "//" + file
if util.is_file_modded(canon, contents, True):
modded_files.append(nest_path)
util.vprint(
f'Found modded file {canon} in {str(name).replace("//", "/")}')
if util.is_file_sarc(canon) and ".ssarc" not in file:
try:
nest_sarc = oead.Sarc(contents)
except ValueError:
continue
sub_mod_files = find_modded_sarc_files(nest_sarc,
name=nest_path,
tmp_dir=tmp_dir,
aoc=aoc)
modded_files.extend(sub_mod_files)
else:
util.vprint(
f'Ignored unmodded file {canon} in {str(name).replace("//", "/")}'
)
return modded_files
def find_modded_sarc_files(mod_sarc: Union[Path, sarc.SARC],
tmp_dir: Path,
name: str = '',
aoc: bool = False,
verbose: bool = False) -> List[str]:
"""
Detects all of the modified files in a SARC
:param mod_sarc: The SARC to scan for modded files.
:type mod_sarc: class:`sarc.SARC`
:param tmp_dir: The path to the base directory of the mod.
:type tmp_dir: class:`pathlib.Path`
:param name: The name of the SARC which contains the current SARC.
:type name: str
:param aoc: Specifies whether the SARC is DLC content, defaults to False.
:type aoc: bool, optional
:param nest_level: The depth to which the current SARC is nested in more SARCs, defaults to 0
:type nest_level: int, optional
:param deep_merge: Whether to log diffs for individual AAMP and BYML files, defaults to False
:type deep_merge: bool, optional
:param verbose: Specifies whether to return more detailed output
:type verbose: bool, optional
"""
if isinstance(mod_sarc, Path):
if any(mod_sarc.name.startswith(exclude) for exclude in ['Bootup_']):
return []
name = str(mod_sarc.relative_to(tmp_dir))
aoc = 'aoc' in mod_sarc.parts or 'Aoc' in mod_sarc.parts
with mod_sarc.open('rb') as s_file:
mod_sarc = sarc.read_file_and_make_sarc(s_file)
if not mod_sarc:
return []
modded_files = []
for file in mod_sarc.list_files():
canon = file.replace('.s', '.')
if aoc:
canon = 'Aoc/0010/' + canon
contents = mod_sarc.get_file_data(file).tobytes()
contents = util.unyaz_if_needed(contents)
nest_path = str(name).replace('\\', '/') + '//' + file
if util.is_file_modded(canon, contents, True):
modded_files.append(nest_path)
if verbose:
print(
f'Found modded file {canon} in {str(name).replace("//", "/")}'
)
if util.is_file_sarc(canon) and '.ssarc' not in file:
try:
nest_sarc = sarc.SARC(contents)
except ValueError:
continue
sub_mod_files = find_modded_sarc_files(nest_sarc,
name=nest_path,
tmp_dir=tmp_dir,
aoc=aoc,
verbose=verbose)
modded_files.extend(sub_mod_files)
else:
if verbose:
print(
f'Ignored unmodded file {canon} in {str(name).replace("//", "/")}'
)
return modded_files
