def merge_sarcs(file_name: str, sarcs: List[Union[Path, bytes]]) -> (str, bytes): opened_sarcs: List[sarc.SARC] = [] if isinstance(sarcs[0], Path): for i, sarc_path in enumerate(sarcs): sarcs[i] = sarc_path.read_bytes() for sarc_bytes in sarcs: sarc_bytes = util.unyaz_if_needed(sarc_bytes) try: opened_sarcs.append(sarc.SARC(sarc_bytes)) except ValueError: continue all_files = {key for open_sarc in opened_sarcs for key in open_sarc.list_files()} nested_sarcs = {} new_sarc = sarc.SARCWriter(be=True) files_added = [] # for file in all_files: # dm_cache = util.get_master_modpack_dir() / 'logs' / 'dm' / file # if dm_cache.exists(): # file_data = dm_cache.read_bytes() # new_sarc.add_file(file, file_data) # files_added.append(file) for opened_sarc in reversed(opened_sarcs): for file in [file for file in opened_sarc.list_files() if file not in files_added]: data = opened_sarc.get_file_data(file).tobytes() if util.is_file_modded(file.replace('.s', '.'), data, count_new=True): if not Path(file).suffix in util.SARC_EXTS: new_sarc.add_file(file, data) files_added.append(file) else: if file not in nested_sarcs: nested_sarcs[file] = [] nested_sarcs[file].append(util.unyaz_if_needed(data)) for file, sarcs in nested_sarcs.items(): merged_bytes = merge_sarcs(file, sarcs)[1] if Path(file).suffix.startswith('.s') and not file.endswith('.sarc'): merged_bytes = util.compress(merged_bytes) new_sarc.add_file(file, merged_bytes) files_added.append(file) for file in [file for file in all_files if file not in files_added]: for opened_sarc in [open_sarc for open_sarc in opened_sarcs \ if file in open_sarc.list_files()]: new_sarc.add_file(file, opened_sarc.get_file_data(file).tobytes()) break if 'Bootup.pack' in file_name: for merger in [merger() for merger in mergers.get_mergers() if merger.is_bootup_injector()]: inject = merger.get_bootup_injection() if not inject: continue file, data = inject try: new_sarc.delete_file(file) except KeyError: pass new_sarc.add_file(file, data) return (file_name, new_sarc.get_bytes())
def _get_sizes_in_sarc(file: Union[Path, sarc.SARC]) -> {}: calc = rstb.SizeCalculator() sizes = {} guess = util.get_settings_bool('guess_merge') if isinstance(file, Path): with file.open('rb') as s_file: file = sarc.read_file_and_make_sarc(s_file) if not file: return {} for nest_file in file.list_files(): canon = nest_file.replace('.s', '.') data = util.unyaz_if_needed(file.get_file_data(nest_file).tobytes()) ext = Path(canon).suffix if util.is_file_modded(canon, data) and ext not in RSTB_EXCLUDE_EXTS and canon not in RSTB_EXCLUDE_NAMES: size = calc.calculate_file_size_with_ext( data, wiiu=True, ext=ext ) if ext == '.bdmgparam': size = 0 if size == 0 and guess: if ext in util.AAMP_EXTS: size = guess_aamp_size(data, ext) elif ext in ['.bfres', '.sbfres']: size = guess_bfres_size(data, canon) sizes[canon] = size if util.is_file_sarc(nest_file) and not nest_file.endswith('.ssarc'): try: nest_sarc = sarc.SARC(data) except ValueError: continue sizes.update(_get_sizes_in_sarc(nest_sarc)) return sizes
def _get_sizes_in_sarc( file: Union[Path, oead.Sarc], guess: bool, is_aoc: bool = False ) -> {}: sizes = {} if isinstance(file, Path): is_aoc = util.get_dlc_path() in file.as_posix() try: file = oead.Sarc(util.unyaz_if_needed(file.read_bytes())) except (RuntimeError, oead.InvalidDataError): print(f"{file} could not be opened") return {} for nest_file, data in [(file.name, file.data) for file in file.get_files()]: canon = nest_file.replace(".s", ".") if data[0:4] == b"Yaz0": data = util.decompress(data) ext = Path(canon).suffix if ( util.is_file_modded(canon, data) and ext not in EXCLUDE_EXTS and canon not in EXCLUDE_NAMES ): sizes[canon] = calculate_size(canon, data, guess=guess) if ext in util.SARC_EXTS - SARC_EXCLUDES: try: nest_sarc = oead.Sarc(data) except (ValueError, RuntimeError, oead.InvalidDataError): continue sizes.update(_get_sizes_in_sarc(nest_sarc, guess, is_aoc=is_aoc)) del nest_sarc del data del file return sizes
def _convert_actorpack(actor_pack: Path, to_wiiu: bool) -> Union[None, str]: error = None sarc = oead.Sarc(util.unyaz_if_needed(actor_pack.read_bytes())) new_sarc = oead.SarcWriter.from_sarc(sarc) new_sarc.set_endianness(oead.Endianness.Big if to_wiiu else oead.Endianness.Little) for file in sarc.get_files(): if "Physics/" in file.name and "Actor/" not in file.name: ext = file.name[file.name.rindex(".") :] if ext in NO_CONVERT_EXTS: if not util.is_file_modded( util.get_canon_name(file.name, allow_no_source=True), file.data, count_new=True, ): actor_name = file.name[ file.name.rindex("/") : file.name.rindex(".") ] try: pack_path = util.get_game_file( f"Actor/Pack/{actor_name}.sbactorpack" ) stock_data = util.get_nested_file_bytes( f"{str(pack_path)}//{file.name}" ) if stock_data: new_sarc.files[file.name] = stock_data else: raise FileNotFoundError(file.name) except (FileNotFoundError, AttributeError): error = ( "This mod contains a Havok file not supported by the " f"converter: {file.name}" ) else: error = ( "This mod contains a Havok file not supported by the" f" converter: {file.name}" ) else: if file.data[0:4] == b"AAMP": continue try: hk = Havok.from_bytes(bytes(file.data)) except: # pylint: disable=bare-except return f"Could not parse Havok file {file.name}" if to_wiiu: hk.to_wiiu() else: hk.to_switch() hk.serialize() new_sarc.files[file.name] = hk.to_bytes() actor_pack.write_bytes(util.compress(new_sarc.write()[1])) return error
def _check_modded(file: Path, tmp_dir: Path): try: canon = util.get_canon_name(file.relative_to(tmp_dir).as_posix()) except ValueError: util.vprint( f"Ignored unknown file {file.relative_to(tmp_dir).as_posix()}") return None if util.is_file_modded(canon, file, True): util.vprint(f"Found modded file {canon}") return file else: if "Aoc/0010/Map/MainField" in canon: file.unlink() util.vprint(f"Ignored unmodded file {canon}") return None
def find_modded_sarc_files(mod_sarc: Union[Path, oead.Sarc], tmp_dir: Path, name: str = "", aoc: bool = False) -> List[str]: if isinstance(mod_sarc, Path): if any(mod_sarc.name.startswith(exclude) for exclude in ["Bootup_"]): return [] name = str(mod_sarc.relative_to(tmp_dir)) aoc = util.get_dlc_path() in mod_sarc.parts or "Aoc" in mod_sarc.parts try: mod_sarc = oead.Sarc(util.unyaz_if_needed(mod_sarc.read_bytes())) except (RuntimeError, ValueError, oead.InvalidDataError): return [] modded_files = [] for file, contents in [(f.name, bytes(f.data)) for f in mod_sarc.get_files()]: canon = file.replace(".s", ".") if aoc: canon = "Aoc/0010/" + canon contents = util.unyaz_if_needed(contents) nest_path = str(name).replace("\\", "/") + "//" + file if util.is_file_modded(canon, contents, True): modded_files.append(nest_path) util.vprint( f'Found modded file {canon} in {str(name).replace("//", "/")}') if util.is_file_sarc(canon) and ".ssarc" not in file: try: nest_sarc = oead.Sarc(contents) except ValueError: continue sub_mod_files = find_modded_sarc_files(nest_sarc, name=nest_path, tmp_dir=tmp_dir, aoc=aoc) modded_files.extend(sub_mod_files) else: util.vprint( f'Ignored unmodded file {canon} in {str(name).replace("//", "/")}' ) return modded_files
def merge_sarcs(file_name: str, sarcs: List[Union[Path, bytes]]) -> (str, bytes): opened_sarcs: List[oead.Sarc] = [] if "ThunderRodLv2" in file_name: print() if isinstance(sarcs[0], Path): for i, sarc_path in enumerate(sarcs): sarcs[i] = sarc_path.read_bytes() for sarc_bytes in sarcs: sarc_bytes = util.unyaz_if_needed(sarc_bytes) try: opened_sarcs.append(oead.Sarc(sarc_bytes)) except (ValueError, RuntimeError, oead.InvalidDataError): continue all_files = { file.name for open_sarc in opened_sarcs for file in open_sarc.get_files() } nested_sarcs = {} new_sarc = oead.SarcWriter(endian=oead.Endianness.Big if util.get_settings( "wiiu") else oead.Endianness.Little) files_added = set() for opened_sarc in reversed(opened_sarcs): for file in [ f for f in opened_sarc.get_files() if f.name not in files_added ]: file_data = oead.Bytes(file.data) if (file.name[file.name.rindex("."):] in util.SARC_EXTS - EXCLUDE_EXTS) and file.name not in SPECIAL: if file.name not in nested_sarcs: nested_sarcs[file.name] = [] nested_sarcs[file.name].append(util.unyaz_if_needed(file_data)) elif util.is_file_modded(file.name.replace(".s", "."), file_data, count_new=True): new_sarc.files[file.name] = file_data files_added.add(file.name) util.vprint(set(nested_sarcs.keys())) for file, sarcs in nested_sarcs.items(): if not sarcs: continue merged_bytes = merge_sarcs(file, sarcs[::-1])[1] if Path(file).suffix.startswith(".s") and not file.endswith(".sarc"): merged_bytes = util.compress(merged_bytes) new_sarc.files[file] = merged_bytes files_added.add(file) for file in [file for file in all_files if file not in files_added]: for opened_sarc in [ open_sarc for open_sarc in opened_sarcs if (file in [f.name for f in open_sarc.get_files()]) ]: new_sarc.files[file] = oead.Bytes(opened_sarc.get_file(file).data) break if "Bootup.pack" in file_name: for merger in [ merger() for merger in mergers.get_mergers() if merger.is_bootup_injector() ]: inject = merger.get_bootup_injection() if not inject: continue file, data = inject new_sarc.files[file] = data return (file_name, bytes(new_sarc.write()[1]))
def find_modded_files(tmp_dir: Path, verbose: bool = False, original_pool: Pool = None) -> List[Union[Path, str]]: """ Detects all of the modified files in an extracted mod :param tmp_dir: The path to the base directory of the mod. :type tmp_dir: class:`pathlib.Path` :param deep_merge: Whether to log diffs for individual AAMP and BYML files, defaults to False :type deep_merge: bool, optional :param verbose: Specifies whether to return more detailed output :type verbose: bool, optional :returns: Returns a tuple with a dict of modified files and the RSTB entries, a list of changes, and (if deep merge) diffs of modded BYML and AAMP files :rtype: (dict of class:`pathlib.Path`: int, list of str, dict of str: str) """ modded_files = [] if isinstance(tmp_dir, str): tmp_dir = Path(tmp_dir) rstb_path = tmp_dir / 'content' / 'System' / 'Resource' /\ 'ResourceSizeTable.product.srsizetable' if rstb_path.exists(): rstb_path.unlink() if (tmp_dir / 'aoc').exists: try: util.get_aoc_dir() except FileNotFoundError as err: err.error_text = ( 'This mod uses DLC files, but you do not appear to have the DLC ' 'installed. If you still want to use this mod, unpack it and ' 'remove the "aoc" folder.') raise err aoc_field = tmp_dir / 'aoc' / '0010' / 'Pack' / 'AocMainField.pack' if aoc_field.exists() and aoc_field.stat().st_size > 0: with aoc_field.open('rb') as a_file: sarc.read_file_and_make_sarc(a_file).extract_to_dir( str(tmp_dir / 'aoc' / '0010')) aoc_field.write_bytes(b'') for file in tmp_dir.rglob('**/*'): if file.is_file(): canon = util.get_canon_name(file.relative_to(tmp_dir).as_posix()) if canon is None: if verbose: print( f'Ignored unknown file {file.relative_to(tmp_dir).as_posix()}' ) continue if util.is_file_modded(canon, file, True): modded_files.append(file) if verbose: print(f'Found modded file {canon}') else: if 'Aoc/0010/Map/MainField' in canon: file.unlink() if verbose: print(f'Ignored unmodded file {canon}') continue total = len(modded_files) print(f'Found {total} modified file{"s" if total > 1 else ""}') total = 0 sarc_files = [ file for file in modded_files if file.suffix in util.SARC_EXTS ] if sarc_files: print(f'Scanning files packed in SARCs...') num_threads = min(len(sarc_files), cpu_count() - 1) pool = original_pool or Pool(processes=num_threads) modded_sarc_files = pool.map( partial(find_modded_sarc_files, tmp_dir=tmp_dir, verbose=verbose), sarc_files) for files in modded_sarc_files: total += len(files) modded_files.extend(files) if not original_pool: pool.close() pool.join() print(f'Found {total} modified packed file{"s" if total > 1 else ""}') return modded_files
def find_modded_sarc_files(mod_sarc: Union[Path, sarc.SARC], tmp_dir: Path, name: str = '', aoc: bool = False, verbose: bool = False) -> List[str]: """ Detects all of the modified files in a SARC :param mod_sarc: The SARC to scan for modded files. :type mod_sarc: class:`sarc.SARC` :param tmp_dir: The path to the base directory of the mod. :type tmp_dir: class:`pathlib.Path` :param name: The name of the SARC which contains the current SARC. :type name: str :param aoc: Specifies whether the SARC is DLC content, defaults to False. :type aoc: bool, optional :param nest_level: The depth to which the current SARC is nested in more SARCs, defaults to 0 :type nest_level: int, optional :param deep_merge: Whether to log diffs for individual AAMP and BYML files, defaults to False :type deep_merge: bool, optional :param verbose: Specifies whether to return more detailed output :type verbose: bool, optional """ if isinstance(mod_sarc, Path): if any(mod_sarc.name.startswith(exclude) for exclude in ['Bootup_']): return [] name = str(mod_sarc.relative_to(tmp_dir)) aoc = 'aoc' in mod_sarc.parts or 'Aoc' in mod_sarc.parts with mod_sarc.open('rb') as s_file: mod_sarc = sarc.read_file_and_make_sarc(s_file) if not mod_sarc: return [] modded_files = [] for file in mod_sarc.list_files(): canon = file.replace('.s', '.') if aoc: canon = 'Aoc/0010/' + canon contents = mod_sarc.get_file_data(file).tobytes() contents = util.unyaz_if_needed(contents) nest_path = str(name).replace('\\', '/') + '//' + file if util.is_file_modded(canon, contents, True): modded_files.append(nest_path) if verbose: print( f'Found modded file {canon} in {str(name).replace("//", "/")}' ) if util.is_file_sarc(canon) and '.ssarc' not in file: try: nest_sarc = sarc.SARC(contents) except ValueError: continue sub_mod_files = find_modded_sarc_files(nest_sarc, name=nest_path, tmp_dir=tmp_dir, aoc=aoc, verbose=verbose) modded_files.extend(sub_mod_files) else: if verbose: print( f'Ignored unmodded file {canon} in {str(name).replace("//", "/")}' ) return modded_files