def log_merged_files_rstb(): """ Generates an RSTB log for the master BCML modpack containing merged files """ print('Updating RSTB for merged files...') diffs = {} files = [item for item in util.get_master_modpack_dir().rglob('**/*') if item.is_file()] guess = util.get_settings_bool('guess_merge') for file in files: if file.parent == 'logs': continue if file.suffix not in RSTB_EXCLUDE_EXTS and file.name not in RSTB_EXCLUDE_NAMES: size = calculate_size(file) if size == 0 and guess: if file.suffix in util.AAMP_EXTS: size = guess_aamp_size(file) elif file.suffix in ['.bfres', '.sbfres']: size = guess_bfres_size(file) canon = util.get_canon_name(file.relative_to(util.get_master_modpack_dir())) if canon: diffs[canon] = size sarc_files = [file for file in files if util.is_file_sarc(str(file)) \ and file.suffix != '.ssarc'] if sarc_files: num_threads = min(multiprocessing.cpu_count(), len(sarc_files)) pool = multiprocessing.Pool(processes=num_threads) results = pool.map(_get_sizes_in_sarc, sarc_files) pool.close() pool.join() for result in results: diffs.update(result) with (util.get_master_modpack_dir() / 'logs' / 'rstb.log').open('w', encoding='utf-8') as log: log.write('name,size,path\n') for canon, size in diffs.items(): log.write(f'{canon},{size},//\n')
def _get_sizes_in_sarc(file: Union[Path, sarc.SARC]) -> {}: calc = rstb.SizeCalculator() sizes = {} guess = util.get_settings_bool('guess_merge') if isinstance(file, Path): with file.open('rb') as s_file: file = sarc.read_file_and_make_sarc(s_file) if not file: return {} for nest_file in file.list_files(): canon = nest_file.replace('.s', '.') data = util.unyaz_if_needed(file.get_file_data(nest_file).tobytes()) ext = Path(canon).suffix if util.is_file_modded(canon, data) and ext not in RSTB_EXCLUDE_EXTS and canon not in RSTB_EXCLUDE_NAMES: size = calc.calculate_file_size_with_ext( data, wiiu=True, ext=ext ) if ext == '.bdmgparam': size = 0 if size == 0 and guess: if ext in util.AAMP_EXTS: size = guess_aamp_size(data, ext) elif ext in ['.bfres', '.sbfres']: size = guess_bfres_size(data, canon) sizes[canon] = size if util.is_file_sarc(nest_file) and not nest_file.endswith('.ssarc'): try: nest_sarc = sarc.SARC(data) except ValueError: continue sizes.update(_get_sizes_in_sarc(nest_sarc)) return sizes
def find_modded_sarc_files(mod_sarc: Union[Path, oead.Sarc], tmp_dir: Path, name: str = "", aoc: bool = False) -> List[str]: if isinstance(mod_sarc, Path): if any(mod_sarc.name.startswith(exclude) for exclude in ["Bootup_"]): return [] name = str(mod_sarc.relative_to(tmp_dir)) aoc = util.get_dlc_path() in mod_sarc.parts or "Aoc" in mod_sarc.parts try: mod_sarc = oead.Sarc(util.unyaz_if_needed(mod_sarc.read_bytes())) except (RuntimeError, ValueError, oead.InvalidDataError): return [] modded_files = [] for file, contents in [(f.name, bytes(f.data)) for f in mod_sarc.get_files()]: canon = file.replace(".s", ".") if aoc: canon = "Aoc/0010/" + canon contents = util.unyaz_if_needed(contents) nest_path = str(name).replace("\\", "/") + "//" + file if util.is_file_modded(canon, contents, True): modded_files.append(nest_path) util.vprint( f'Found modded file {canon} in {str(name).replace("//", "/")}') if util.is_file_sarc(canon) and ".ssarc" not in file: try: nest_sarc = oead.Sarc(contents) except ValueError: continue sub_mod_files = find_modded_sarc_files(nest_sarc, name=nest_path, tmp_dir=tmp_dir, aoc=aoc) modded_files.extend(sub_mod_files) else: util.vprint( f'Ignored unmodded file {canon} in {str(name).replace("//", "/")}' ) return modded_files
def find_modded_sarc_files(mod_sarc: Union[Path, sarc.SARC], tmp_dir: Path, name: str = '', aoc: bool = False, verbose: bool = False) -> List[str]: """ Detects all of the modified files in a SARC :param mod_sarc: The SARC to scan for modded files. :type mod_sarc: class:`sarc.SARC` :param tmp_dir: The path to the base directory of the mod. :type tmp_dir: class:`pathlib.Path` :param name: The name of the SARC which contains the current SARC. :type name: str :param aoc: Specifies whether the SARC is DLC content, defaults to False. :type aoc: bool, optional :param nest_level: The depth to which the current SARC is nested in more SARCs, defaults to 0 :type nest_level: int, optional :param deep_merge: Whether to log diffs for individual AAMP and BYML files, defaults to False :type deep_merge: bool, optional :param verbose: Specifies whether to return more detailed output :type verbose: bool, optional """ if isinstance(mod_sarc, Path): if any(mod_sarc.name.startswith(exclude) for exclude in ['Bootup_']): return [] name = str(mod_sarc.relative_to(tmp_dir)) aoc = 'aoc' in mod_sarc.parts or 'Aoc' in mod_sarc.parts with mod_sarc.open('rb') as s_file: mod_sarc = sarc.read_file_and_make_sarc(s_file) if not mod_sarc: return [] modded_files = [] for file in mod_sarc.list_files(): canon = file.replace('.s', '.') if aoc: canon = 'Aoc/0010/' + canon contents = mod_sarc.get_file_data(file).tobytes() contents = util.unyaz_if_needed(contents) nest_path = str(name).replace('\\', '/') + '//' + file if util.is_file_modded(canon, contents, True): modded_files.append(nest_path) if verbose: print( f'Found modded file {canon} in {str(name).replace("//", "/")}' ) if util.is_file_sarc(canon) and '.ssarc' not in file: try: nest_sarc = sarc.SARC(contents) except ValueError: continue sub_mod_files = find_modded_sarc_files(nest_sarc, name=nest_path, tmp_dir=tmp_dir, aoc=aoc, verbose=verbose) modded_files.extend(sub_mod_files) else: if verbose: print( f'Ignored unmodded file {canon} in {str(name).replace("//", "/")}' ) return modded_files