def PackMod_External(mod_name, target): util.InitialiseLog(target + ".log") util.StartTimer() util.LogInfo("Skyrim-NX-Toolkit {} - pack_mod".format( util.GetToolkitVersion())) PackMod(mod_name, target) util.EndTimer()
def LoadOrder_External(origin, target, loadOrderName): util.InitialiseLog(os.path.join(origin, loadOrderName) + ".log") util.StartTimer() util.LogInfo("Skyrim-NX-Toolkit {} - load_order".format( util.GetToolkitVersion())) LoadOrder(origin, target, loadOrderName) util.EndTimer()
def RepackMod_External(origin, target): util.InitialiseLog(origin + ".log") util.StartTimer() util.LogInfo("Skyrim-NX-Toolkit {} - repack_mod".format( util.GetToolkitVersion())) RepackMod(origin, target) util.EndTimer()
def WriteBSA(): nonlocal currentFileIndex, bsa_filename util.LogDebug("Writing BSA") if currentFileIndex != None: bsa_filename = bsa_original_filename[:-4] + str( currentFileIndex) + ".bsa" currentFileIndex += 1 util.LogInfo("Run bsarch.exe") flags_value = flags.GetValue() flags_hexvalue = hex(flags_value) compress = "-z" if flags.IsSet(Flag_Compressed) else "" commandLine = [ bsarch, "pack", temp_data, bsa_filename, "-sse", compress, "-af:" + flags_hexvalue ] util.RunCommandLine(commandLine) bsaFileWritten.append({ "Folder": target_folder, "FileName": bsa_filename }) util.RemoveTree(temp_data)
def NXOPUS2FUZ(filename_nxopus, channel_count, sound_duration_ms, sound_file): """ adds required NX SSE header to a nxopus sound """ try: with open(filename_nxopus, "rb") as nxopus_file: nxopus_data = bytearray(nxopus_file.read()) nxopus_size = len(nxopus_data) except: util.LogInfo("ERROR: failed to open intermediate NXOPUS <{}>.".format( filename_nxopus)) return True nx_sse_opus_header_size = b'\x14\x00\x00\x00' nx_sse_opus_signature = b'\x0A\x8D\xD5\xFF' sound_file.write(nx_sse_opus_signature) sound_file.write( sound_duration_ms.to_bytes(4, byteorder='little', signed=False)) sound_file.write( channel_count.to_bytes(4, byteorder='little', signed=False)) sound_file.write(nx_sse_opus_header_size) sound_file.write(nxopus_size.to_bytes(4, byteorder='little', signed=False)) sound_file.write(nxopus_data) return False
def ConvertPath_External(mod_name, target): util.InitialiseLog(target + ".log") util.StartTimer() util.LogInfo("Skyrim-NX-Toolkit {} - convert_path".format( util.GetToolkitVersion())) ConvertPath(mod_name, target) util.EndTimer()
def ConvertHKX64_Internal(filename): util.LogDebug("ConvertHKX64 : " + filename) with open(filename, "rb") as hkx64: payload = hkx64.read() header_location = payload.find(b'\x00' * 15 + b'\x80' + b'\x00' * 48) is_skeleton_container = payload.find(b'hkaSkeleton') > 0 is_behavior_container = payload.find(b'hkbBehavior') > 0 # it should not happen here as convert_path already doing some checks if is_behavior_container or is_skeleton_container: util.LogInfo("Warning, cannot convert {} because <{}>".format( filename, 'SSE hkx animation')) return False # swap some fields in the AMD64 HKX header structure to make it PS4 compatible if header_location > 0: is_project_data = payload.find(b'hkbProjectData') offset = 72 if is_project_data > 0 else 76 with open(filename, "wb") as hkxPS4: hkxPS4.write(payload[0:18]) hkxPS4.write(b'\x01') hkxPS4.write(payload[19:header_location + 16]) hkxPS4.write(payload[header_location + 20:header_location + offset]) hkxPS4.write(payload[header_location:header_location + 4]) hkxPS4.write(payload[header_location + offset:]) return True
def ConvertMod_External(origin, target, oldrim): print("<{}>".format(origin)) util.InitialiseLog(origin + ".log") util.StartTimer() util.LogInfo("Skyrim-NX-Toolkit {} - convert_mod".format(util.GetToolkitVersion())) ConvertMod(origin, target, oldrim) util.EndTimer()
def WrtiteBSA(): nonlocal currentFileIndex, buffer, bsa_filename util.LogDebug("Writing BSA with filelist:<" + buffer + ">") filelist_basename = "bsa_filelist.txt" if currentFileIndex != None: filelist_basename = "bsa_filelist" + str(currentFileIndex) + ".txt" bsa_filename = bsa_original_filename[:-4] + str( currentFileIndex) + ".bsa" currentFileIndex += 1 filelist_filename = os.path.join(target_folder, filelist_basename) with open(filelist_filename, 'w') as filelist_file: filelist_file.write(buffer) buffer = '' util.LogInfo("Build Config") checksOrder = [ "Meshes", "Textures", "Menus", "Sounds", "Voices", "Shaders", "Trees", "Fonts", "Misc", "Compress Archive", "Retain Directory Names", "Retain File Names", "Retain File Name Offsets", "Retain Strings During Startup", "XBox 360 Archive", "Embed File Names" ] with open(config_filename, 'w') as config_file: config_file.write("Log: " + log_basename + "\n") config_file.write("New Archive\n") for check in checksOrder: if check in checks: config_file.write("Check: " + check + "\n") config_file.write("Set File Group Root: Data\\\n") config_file.write("Add File Group: " + filelist_basename + "\n") config_file.write("Save Archive: " + bsa_filename + "\n") util.LogInfo("Run Archive.exe") commandLine = ["Archive.exe", config_basename] os.chdir(target_folder) util.RunCommandLine(commandLine) with open(log_filename, "r") as log_file: for line in log_file: util.LogDebug(line) os.remove(log_filename) os.remove(filelist_filename) os.remove(config_filename) bsaFileWritten.append({ "Folder": target_folder, "FileName": bsa_filename })
def DSP2MCADPCM(filename_dsp0, filename_dsp1, channel_count, sound_file): """ creates a MCADPCM file from DSPADPCM ones """ try: with open(filename_dsp0, "rb") as dsp0_file: dsp0_data = bytearray(dsp0_file.read()) dsp0_size = len(dsp0_data) DSP2LITTLE_ENDIAN(dsp0_data, 0x00) except: util.LogInfo("ERROR: failed to open intermediate DSP <{}>.".format( filename_dsp0)) return True if channel_count > 1: try: with open(filename_dsp1, "rb") as dsp1_file: dsp1_data = bytearray(dsp1_file.read()) dsp1_size = len(dsp1_data) dsp1_offset = 0x14 + dsp0_size DSP2LITTLE_ENDIAN(dsp1_data, 0x00) except: util.LogInfo("ERROR: failed to open intermediate DSP <{}>.".format( filename_dsp1)) return True header_stereo = b'\x02\x00\x00\x00\x14\x00\x00\x00' sound_file.write(header_stereo) sound_file.write( dsp0_size.to_bytes(4, byteorder='little', signed=False)) sound_file.write( dsp1_offset.to_bytes(4, byteorder='little', signed=False)) sound_file.write( dsp1_size.to_bytes(4, byteorder='little', signed=False)) sound_file.write(dsp0_data) sound_file.write(dsp1_data) else: header_single = b'\x01\x00\x00\x00\x0C\x00\x00\x00' sound_file.write(header_single) sound_file.write( dsp0_size.to_bytes(4, byteorder='little', signed=False)) sound_file.write(dsp0_data) return False
def ConvertTXT_Internal(filename): buffer = None with open(filename, "rb") as pcFile: buffer = pcFile.read() buffer = bytearray(buffer) util.LogInfo("Buffer0<{}>".format(buffer[0])) util.LogInfo("Buffer0<{}>".format(buffer[1])) i = 1 while i < len(buffer): buffer[i:i] = b'\x00\x00\x00' i += 4 util.LogDebug("BUFFER<{}>".format(buffer)) with open(filename, "wb") as outFile: outFile.write(b'\xff\xfe\x00\x00') outFile.write(buffer) return True
def ConvertMod(origin, target, oldrim): mod_name = os.path.basename(origin) ''' util.LogDebug("This is the origin: " + origin) util.LogDebug("This is the target: " + target) util.LogDebug("This is the mod name " + mod_name) ''' util.LogDebug("convert_mod.py 2.0") toolkit_path = util.GetToolKitPath() util.LogInfo("Convert Mod, ToolkitPath is {}".format(toolkit_path)) util.LogInfo("Convert Mod, create empty folder at target") util.CreateTarget(target) unpack_mod.UnpackMod(origin, target) if oldrim: reconcile_hkx.ReconcileHKX(target, oldrim) convert_path.ConvertPath(mod_name, target) pack_mod.PackMod(mod_name, target)
def ReconcileHKX(mod_path, oldrim_path): script_path = util.GetScriptPath() util.LogInfo("Reconcile HKX") util.LogDebug("This is the mod_path: " + mod_path) util.LogDebug("This is the oldrim_path " + oldrim_path) CopyHKX = [] totalCount = 0 matchedCount = 0 for root, subdirs, files in os.walk(mod_path): util.LogDebug("Walking folder " + root) for filename in files: if filename.lower().endswith(".hkx"): file_path = os.path.join(root, filename) relative_path = os.path.relpath(file_path, mod_path) util.LogDebug("Relative path {} OR Path {}".format( relative_path, oldrim_path)) oldrim_file_path = os.path.join(oldrim_path, relative_path) totalCount += 1 util.LogDebug("Found {}, checking {}".format( file_path, oldrim_file_path)) if os.path.exists(oldrim_file_path): util.LogDebug( "Found {} match in oldrim".format(oldrim_file_path)) matchedCount += 1 CopyHKX.append((file_path, oldrim_file_path)) util.LogInfo("Matched {}/{} hkx files in the mod".format( matchedCount, totalCount)) for i in range(len(CopyHKX)): (copy_to, copy_from) = CopyHKX[i] util.LogDebug("Copying {}->{}".format(copy_from, copy_to)) shutil.copy2(copy_from, copy_to) sys.stdout.write("Reconciled {}/{} \r".format(i + 1, len(CopyHKX))) sys.stdout.flush() sys.stdout.write("\n")
def get_setting(section, setting): """ Print out a setting """ config = get_config() try: value = config.get(section, setting) except: value = DefaultValues[section][setting] util.LogInfo("Added default setting {}/{} as {}".format( section, setting, value)) update_setting(section, setting, str(value)) return value
def RepackMod(origin, target): mod_name = os.path.basename(origin) ''' util.LogDebug("This is the origin: " + origin) util.LogDebug("This is the target: " + target) util.LogDebug("This is the mod name " + mod_name) ''' util.LogDebug("convert_mod.py 2.0") util.LogInfo("Convert Mod, create empty folder at target") util.CreateTarget(target) unpack_mod.UnpackMod(origin, target) pack_mod.PackMod(mod_name, target)
def check_clear(): config = get_config() util.LogInfo("Checking if settings.ini should be updated.") clearOldConfig = False currentVersion = util.GetToolkitVersion() try: settingsVersion = config.get("Version", "ToolkitVersion") util.LogInfo( "Version:ToolkitVersion is {}. Currrent Version is {}".format( settingsVersion, currentVersion)) if settingsVersion != currentVersion: util.LogInfo("Not a match. Clearing settings.ini") clearOldConfig = True except: util.LogInfo( "Version:ToolkitVersion was not found. Clearing settings.ini") clearOldConfig = True if clearOldConfig: util.LogInfo("Clearing settings.ini") toolkit_path = util.GetToolKitPath() path = os.path.join(toolkit_path, "settings.ini") create_config(path) config.read(path)
while i < len(buffer): buffer[i:i] = b'\x00\x00\x00' i += 4 util.LogDebug("BUFFER<{}>".format(buffer)) with open(filename, "wb") as outFile: outFile.write(b'\xff\xfe\x00\x00') outFile.write(buffer) return True def ConvertTXT(target, filename): return ConvertTXT_Internal(filename) def ConvertTXTAsync(target, filename, logname, ret): util.InitialiseMPLog(logname) retVal = ConvertTXT(target, filename) ret["retVal"] = retVal if __name__ == '__main__': filename = sys.argv[1] util.InitialiseLog(filename + ".log") util.StartTimer() util.LogInfo("Skyrim-NX-Toolkit {} - convert_txt".format( util.GetToolkitVersion())) ConvertTXT_Internal(filename) util.EndTimer()
def BsarchBSA(target_folder, bsa_filename): script_path = util.GetScriptPath() utilities_path = util.GetUtilitiesPath() bsarch = os.path.join(utilities_path, "bsarch.exe") log_basename = "log.txt" log_filename = os.path.join(target_folder, log_basename) config_basename = "bsa_config.txt" config_filename = os.path.join(target_folder, config_basename) allFilesList = [] Flag_NamedDir = 1 Flag_NamedFiles = 2 Flag_Compressed = 4 Flag_RetainDir = 8 Flag_RetainName = 16 Flag_RetainFOff = 32 Flag_XBox360 = 64 Flag_StartupStr = 128 Flag_EmbedName = 256 Flag_XMem = 512 Flag_Bit = 1024 flags = bitflag.BitFlag() flags.SetFlag(Flag_NamedDir) flags.SetFlag(Flag_NamedFiles) util.LogInfo("Build File List") totalFileSizeTally = 0 target_data = os.path.join(target_folder, "Data") util.LogDebug("Walking the target directory " + target_data) bsaFolders = [] SizeLimitBSA = bsa_rules.BSASizeLimit totalFileCount = 0 for root, subdirs, files in os.walk(target_data): util.LogDebug('--\nroot = ' + root) if root != target_data: for filename in files: if filename != "desktop.ini" and filename != 'thumbs.db': file_path = os.path.join(root, filename) file_size = os.path.getsize(file_path) totalFileSizeTally += file_size totalFileCount += 1 currentFileIndex = None if totalFileSizeTally > SizeLimitBSA: currentFileIndex = 0 totalWrittenTally = 0 currentFileSizeTally = 0 buffer = '' bsaFileWritten = [] bsa_original_filename = bsa_filename temp_data = os.path.join(os.path.dirname(target_data), "Ready") def WriteBSA(): nonlocal currentFileIndex, bsa_filename util.LogDebug("Writing BSA") if currentFileIndex != None: bsa_filename = bsa_original_filename[:-4] + str( currentFileIndex) + ".bsa" currentFileIndex += 1 util.LogInfo("Run bsarch.exe") flags_value = flags.GetValue() flags_hexvalue = hex(flags_value) compress = "-z" if flags.IsSet(Flag_Compressed) else "" commandLine = [ bsarch, "pack", temp_data, bsa_filename, "-sse", compress, "-af:" + flags_hexvalue ] util.RunCommandLine(commandLine) bsaFileWritten.append({ "Folder": target_folder, "FileName": bsa_filename }) util.RemoveTree(temp_data) filesArchived = 0 for root, subdirs, files in os.walk(target_data): util.LogDebug('--\nroot = ' + root) if root == target_data: util.LogDebug("subdirs: " + str(subdirs)) lower_case_data_list = [x.lower() for x in subdirs] util.LogDebug("lcds: " + str(lower_case_data_list)) if "meshes" in lower_case_data_list: util.LogDebug("found meshes") flags.SetFlag(Flag_StartupStr) flags.SetFlag(Flag_Compressed) if "textures" in lower_case_data_list: util.LogDebug("found textures") if "interface" in lower_case_data_list: util.LogDebug("found interface") if "music" in lower_case_data_list: util.LogDebug("found music") flags.SetFlag(Flag_RetainName) if "sound" in lower_case_data_list: util.LogDebug("found sound") sound_list = os.listdir(os.path.join(target_data, "sound")) sound_list_lower = [x.lower() for x in sound_list] if "fx" in sound_list_lower: util.LogDebug("found sound//fx") flags.SetFlag(Flag_RetainName) if "voice" in sound_list_lower: util.LogDebug("found sound//voice") if "shadersfx" in lower_case_data_list: util.LogDebug("found shaders") if "seq" in lower_case_data_list: util.LogDebug("found seq") flags.SetFlag(Flag_RetainName) if "grass" in lower_case_data_list: util.LogDebug("found grass") flags.SetFlag(Flag_RetainName) if "scripts" in lower_case_data_list: util.LogDebug("found scripts") flags.SetFlag(Flag_RetainName) else: for filename in files: if filename != "desktop.ini" and filename != 'thumbs.db': file_path = os.path.join(root, filename) file_size = os.path.getsize(file_path) newTally = currentFileSizeTally + file_size util.LogDebug("Attempting to add " + file_path + " currentFileSizeTally is " + str(currentFileSizeTally) + " file_size is " + str(file_size)) if (newTally >= SizeLimitBSA): util.LogDebug( "New BSA would be too big, writing current BSA") sys.stdout.write("\n") WriteBSA() currentFileSizeTally = 0 relative_path = file_path.replace(target_folder, '') path_no_data = relative_path[6:] temp_path = os.path.join(temp_data, path_no_data) util.LogDebug("Moving <{}> to <{}>".format( file_path, temp_path)) paths_to_create = [] check_path = os.path.dirname(temp_path) util.LogDebug("Checking path {}".format(check_path)) while not os.path.isdir(check_path): util.LogDebug("{} does not exist.".format(check_path)) paths_to_create.insert(0, check_path) check_path = os.path.dirname(check_path) util.LogDebug("Checking path {}".format(check_path)) for path_to_create in paths_to_create: util.LogDebug("{} does not exist.".format(check_path)) os.mkdir(path_to_create) shutil.move(file_path, temp_path) currentFileSizeTally += file_size filesArchived += 1 sys.stdout.write("Prepared {}/{} \r".format( filesArchived, totalFileCount)) sys.stdout.flush() sys.stdout.write("\n") if currentFileSizeTally > 0: WriteBSA() util.LogInfo("Clean Up") os.chdir(script_path) return bsaFileWritten
def ConvertPath(mod_name, target): script_path = util.GetScriptPath() util.LogInfo("Convert Path") util.LogDebug("This is the target: " + target) util.LogDebug("This is the mod name " + mod_name) has_havoc = util.HasHavokBPP() do_meshes = \ toolkit_config.get_bool_setting("Meshes", "RemoveEditorMarker") or \ toolkit_config.get_bool_setting("Meshes", "PrettySortBlocks") or \ toolkit_config.get_bool_setting("Meshes", "TrimTexturesPath") or \ toolkit_config.get_bool_setting("Meshes", "OptimizeForSSE") NoConversion = {} WarnConversion = {} ConvertListDDS = [] ConvertListHKX = [] ConvertListHKX64 = [] ConvertListTXT = [] ConvertListSound = [] ConvertListMesh = [] for root, subdirs, files in os.walk(target): if root != target: util.LogDebug("Walking folder " + root) for filename in files: #util.LogDebug("filename: {}".format(filename)) if filename.lower().endswith(".dds"): file_path = os.path.join(root, filename) ddsChecked = check_dds.CheckDDS(file_path, file_path) if ddsChecked == check_dds.PC: ConvertListDDS.append(file_path) elif ddsChecked == check_dds.NX: if 'DDS' not in NoConversion: NoConversion['DDS'] = 0 NoConversion['DDS'] += 1 else: if 'DDS' not in WarnConversion: WarnConversion['DDS'] = [] WarnConversion['DDS'].append( (filename, "Unknown DDS format")) elif filename.lower().endswith(".hkx"): file_path = os.path.join(root, filename) hkxChecked = check_hkx.CheckHKX(file_path, file_path) if hkxChecked == check_hkx.PC_32: ConvertListHKX.append(file_path) elif hkxChecked == check_hkx.PC_XML: ConvertListHKX.append(file_path) elif hkxChecked == check_hkx.PC_64: # if root.lower().find("behaviors") > 0 or filename.lower().find("skeleton") > 0: # if 'HKX' not in WarnConversion: # WarnConversion['HKX'] = [] # WarnConversion['HKX'].append( (filename, "SSE hkx animation") ) # else: ConvertListHKX64.append(file_path) elif hkxChecked == check_hkx.NX_64: if 'HKX' not in NoConversion: NoConversion['HKX'] = 0 NoConversion['HKX'] += 1 else: if 'HKX' not in WarnConversion: WarnConversion['HKX'] = [] WarnConversion['HKX'].append( (filename, "Unknown hkx animation format")) elif filename.lower().startswith( "translate_") and filename.lower().endswith(".txt"): file_path = os.path.join(root, filename) ConvertListTXT.append(file_path) elif filename.lower().endswith(".xwm") or filename.lower( ).endswith(".fuz") or filename.lower().endswith(".wav"): file_path = os.path.join(root, filename[:-4]) if not file_path in ConvertListSound: ConvertListSound.append(file_path) elif filename.lower().endswith(".nif"): file_path = os.path.join(root, filename) ConvertListMesh.append(file_path) for fileType in NoConversion: util.LogInfo("Found {} {} files that are already in NX format".format( NoConversion[fileType], fileType)) for fileType in WarnConversion: fileTypeWarnings = WarnConversion[fileType] for i in range(len(fileTypeWarnings)): util.LogInfo("Warning, cannot convert {} because <{}>".format( fileTypeWarnings[i][0], fileTypeWarnings[i][1])) util.LogInfo("Found {} dds files to convert".format(len(ConvertListDDS))) if has_havoc: util.LogInfo("Found {} 32-bit hkx files to convert".format( len(ConvertListHKX))) else: util.LogInfo( "Found {} 32-bit hkx files that won't convert as Havoc utility wasn't found." .format(len(ConvertListHKX))) util.LogInfo("Found {} 64-bit hkx files to convert".format( len(ConvertListHKX64))) util.LogInfo("Found {} txt files to convert".format(len(ConvertListTXT))) util.LogInfo("Found {} sound files to convert".format( len(ConvertListSound))) if do_meshes: util.LogInfo("Found {} mesh files to convert".format( len(ConvertListMesh))) else: util.LogInfo("Found {} mesh files but won't touch them.".format( len(ConvertListMesh))) ''' def LogProgress(convertList, convertFn, name): if len(convertList) > 0: failedCount = 0 for i in range(len(convertList)): file_path = convertList[i] success = convertFn(target, file_path) if not success: failedCount += 1 sys.stdout.write("Converted {}/{} {} ({}) failed. \r".format(i+1, len(convertList), name, failedCount)) sys.stdout.flush() sys.stdout.write("\n") ''' def LogProgress(convertList, fnName, convertFn, name, threadSetting): if len(convertList) > 0: failedCount = 0 maxThreads = toolkit_config.get_int_setting( "Performance", threadSetting) jm = job_manager.JobManager(maxThreads) convertedCount = 0 processedCount = 0 totalCount = len(convertList) def cb(success): nonlocal processedCount, convertedCount, failedCount processedCount += 1 if success: convertedCount += 1 else: failedCount += 1 sys.stdout.write( "{} Processed {}/{} ({}/{}) success/failure. \r".format( name, processedCount, totalCount, convertedCount, failedCount)) sys.stdout.flush() for i in range(len(convertList)): file_path = convertList[i] job = job_manager.Job(cb, fnName, convertFn, target.lower(), file_path.lower()) jm.AddJob(job) jm.ProcessBatch() sys.stdout.write( "{} Processing Complete: {}/{} ({}/{}) success/failure. \r". format(name, processedCount, totalCount, convertedCount, failedCount)) sys.stdout.write("\n") if processedCount != totalCount: sys.stdout.write("Not all were processed.\n") sys.stdout.flush() LogProgress(ConvertListDDS, "ConvertDDS", convert_dds.ConvertDDS, "DDS", "MaxTextureThreads") if has_havoc: LogProgress(ConvertListHKX, "ConvertHKX", convert_hkx.ConvertHKX, "HKX 32-bit", "MaxAnimationThreads") LogProgress(ConvertListHKX64, "ConvertHKX64", convert_hkx64.ConvertHKX64, "HKX 64-bit", "MaxAnimationThreads") LogProgress(ConvertListTXT, "ConvertTXT", convert_txt.ConvertTXT, "TXT", "MaxOtherThreads") LogProgress(ConvertListSound, "ConvertSound", convert_sound.ConvertSound, "Sounds", "MaxSoundThreads") if do_meshes: LogProgress(ConvertListMesh, "ConvertMesh", convert_nif.ConvertNIF, "Meshes", "MaxMeshThreads")
def WAV2PCM16WAV(filename_xwm, filename_wav, isNxOpus): """ Normalizes the WAV file to be a proper PCM16 with correct sample rate for VGAudioCli """ try: with open(filename_wav, "rb") as wav_file: wav_header = wav_file.read(0xFF) wav_format = wav_header[0x08:0x0C].decode() except: util.LogInfo("ERROR: failed to create intermediate WAV <{}>.".format( filename_wav)) return (True, None, None) # check for a collateral case where Mod Authors save XWM with WAV extension if wav_format == "XWMA": util.LogInfo( "WARNING: <{}> has WAV extension but is a XWMA. Fixing it.".format( filename_wav)) filename_temp = filename_wav + "temp.wav" util.RemoveFile(filename_temp) util.RenameFile(filename_wav, filename_temp) xWMAEncode = GetxWMAEncode() commandLine = [xWMAEncode, filename_temp, filename_wav] util.RunCommandLine(commandLine) util.RemoveFile(filename_temp) try: with open(filename_wav, "rb") as wav_file: wav_header = wav_file.read(0xFF) wav_format = wav_header[0x08:0x0C].decode() except: util.LogInfo( "ERROR: failed to fix collateral case on WAV <{}>.".format( filename_wav)) return (True, None, None) wav_audio_format = int.from_bytes(wav_header[0x14:0x16], byteorder='little', signed=False) wav_sample_rate = int.from_bytes(wav_header[0x18:0x1C], byteorder='little', signed=False) wav_bits_per_sample = int.from_bytes(wav_header[0x22:0x24], byteorder='little', signed=False) # # make the WAV file compatible with VGAudioCLi # # OPUS CODEC requires 24000 or 48000 sample rates from a PCM16 stream # DSPADPCM CODEC requires 22050 or 44100 sample rates from a PCM16 stream # # get the closest ressampling RATE DESIRED_RATES = [24000, 48000] if isNxOpus else [22050, 44100] try: i = 0 while wav_sample_rate >= DESIRED_RATES[i]: i += 1 except: i = 1 override_sample_rate = str(DESIRED_RATES[i]) # ressample if required if (isNxOpus and not wav_sample_rate in DESIRED_RATES ) or wav_audio_format != 1 or wav_bits_per_sample != 16: filename_temp = filename_wav + "temp.wav" util.RemoveFile(filename_temp) util.RenameFile(filename_wav, filename_temp) SndFileConvert = GetSndFileConvert() commandLine = [ SndFileConvert, "-override-sample-rate=" + override_sample_rate, "-pcm16", filename_temp, filename_wav ] util.RunCommandLine(commandLine) util.RemoveFile(filename_temp) # really need to open one last time to get the metadata right try: with open(filename_wav, "rb") as wav_file: wav_header = wav_file.read(0xFF) wav_sample_rate = int.from_bytes(wav_header[0x18:0x1C], byteorder='little', signed=False) wav_bits_per_sample = int.from_bytes(wav_header[0x22:0x24], byteorder='little', signed=False) except: util.LogInfo("ERROR: failed to create normalized WAV <{}>.".format( filename_wav)) return (True, None, None) wav_channel_count = int.from_bytes(wav_header[0x16:0x18], byteorder='little', signed=False) try: data_size_offset = wav_header.find(b'\x64\x61\x74\x61') + 0x04 wav_data_size = int.from_bytes( wav_header[data_size_offset:data_size_offset + 0x04], byteorder='little', signed=False) wav_duration_ms = int( float(wav_data_size * 1000) / float( wav_sample_rate * wav_channel_count * wav_bits_per_sample / 8)) except: wav_duration_ms = 0 util.LogDebug( "Converted WAV <{}>:\n DURATION:{} FORMAT:{} CHANNELS:{} SAMPLE_RATE:{} BITS_PER_SAMPLE:{} DATA_SIZE:{}" .format(filename_wav, wav_duration_ms, wav_audio_format, wav_channel_count, wav_sample_rate, wav_bits_per_sample, wav_data_size)) return (False, wav_channel_count, wav_duration_ms)
def ConvertSound_Internal(filepath_without_extension): """ Converts PC SSE sound files to be compatible with NX SSE supported codecs """ filename_mcadpcm = filepath_without_extension + ".mcadpcm" filename_nxopus = filepath_without_extension + ".lopus" filename_dsp0 = filepath_without_extension + "_CH0_.dsp" filename_dsp1 = filepath_without_extension + "_CH1_.dsp" filename_wav = filepath_without_extension + ".wav" filename_xwm = filepath_without_extension + ".xwm" filename_lip = filepath_without_extension + ".lip" filename_fuz = filepath_without_extension + ".fuz" has_wav = os.path.exists(filename_wav) has_xwm = os.path.exists(filename_xwm) has_lip = os.path.exists(filename_lip) has_fuz = os.path.exists(filename_fuz) # get the desired CODEC if "\\music\\" in filepath_without_extension.lower(): codec = toolkit_config.get_setting("Sounds", "music") elif "\\sound\\fx\\" in filepath_without_extension.lower(): codec = toolkit_config.get_setting("Sounds", "fx") elif "\\sound\\voice\\" in filepath_without_extension.lower(): codec = toolkit_config.get_setting("Sounds", "voice") else: codec = "nxopus" is_nxopus = codec.lower() == "nxopus" util.LogDebug( "Convert Sound <{}> WAV:{} XWM:{} LIP:{} FUZ:{} CODEC:{}".format( filepath_without_extension, has_wav, has_xwm, has_lip, has_fuz, codec)) # FUZ files always have precedence over loose WAV, XWM or LIP lip_size = 0 if has_fuz: try: with open(filename_fuz, "rb") as fuz_file: fuz_file.seek(0x08) lip_size = int.from_bytes(fuz_file.read(0x04), byteorder='little', signed=False) lip_data = fuz_file.read(lip_size) if lip_size == 0: util.LogDebug( "INFO: FUZ {} has a 0 bytes LIP data.".format( filename_fuz)) try: with open(filename_xwm, "wb") as xwm_file: xwm_file.write(fuz_file.read()) has_xwm = True except: util.LogInfo( "ERROR: failed to create intermediate WMV <{}>.". format(filename_xwm)) return False util.LogDebug("INFO: XWM created on disk from FUZ {}.".format( filename_fuz)) except: util.LogInfo( "ERROR: failed to open FUZ <{}>.".format(filename_lip)) return False # Load LIP from disk in the rare case the LIP in FUZ is empty if lip_size == 0 and has_lip: try: with open(filename_lip, "rb") as lip_file: lip_data = lip_file.read() lip_size = len(lip_data) except: util.LogInfo( "ERROR: failed to open LIP <{}>.".format(filename_lip)) return False # Convert the XWM to WAV if has_xwm: XWM2WAV(filename_xwm, filename_wav) # Normalizes the WAV format (err, channel_count, sound_duration_ms) = WAV2PCM16WAV(filename_xwm, filename_wav, is_nxopus) if err: return False # Converte the WAV to NXOPUS or MCADPCM if is_nxopus: WAV2NXOPUS(filename_wav, filename_nxopus) else: WAV2DSP(filename_wav, filename_dsp0, filename_dsp1, channel_count) # write a FUZ container if lip_size > 0 or is_nxopus: lip_padding = lip_size % 4 if lip_padding != 0: lip_padding = 4 - lip_padding voice_offset = 0x10 + lip_size + lip_padding # write the FUZ header # # on PC a FUZ header has 0x0A bytes. NX adds an additional uint32_t to header end # it represents the offset where the SOUND content starts as NX pads LIP content # fuz_nx_payload = BytesIO() header_fuz = b'\x46\x55\x5A\x45\x01\x00\x00\x00' fuz_nx_payload.write(header_fuz) fuz_nx_payload.write( lip_size.to_bytes(4, byteorder='little', signed=False)) fuz_nx_payload.write( voice_offset.to_bytes(4, byteorder='little', signed=False)) # write the LIP content if lip_size > 0: fuz_nx_payload.write(lip_data) fuz_nx_payload.write(b'\x00' * lip_padding) # write the SOUND content if is_nxopus: err = NXOPUS2FUZ(filename_nxopus, channel_count, sound_duration_ms, fuz_nx_payload) else: err = DSP2MCADPCM(filename_dsp0, filename_dsp1, channel_count, fuz_nx_payload) if err: return False # pad the FUZ content to the closest uint32 fuz_padding = fuz_nx_payload.getbuffer().nbytes % 4 if fuz_padding != 0: fuz_padding = 4 - fuz_padding fuz_nx_payload.write(b'\x00' * fuz_padding) try: with open(filename_fuz, "wb") as fuz_nx_file: fuz_nx_file.write(fuz_nx_payload.getbuffer()) except: util.LogInfo( "ERROR: failed to create final FUZ <{}>.".format(filename_fuz)) return False # write a MCADPCM container else: try: with open(filename_mcadpcm, "wb") as mcadpcm_file: DSP2MCADPCM(filename_dsp0, filename_dsp1, channel_count, mcadpcm_file) except: util.LogInfo("ERROR: failed to create final MCADPCM <{}>.".format( filename_mcadpcm)) return False # clean up temporary files util.RemoveFile(filename_wav) if has_xwm: util.RemoveFile(filename_xwm) if has_lip: util.RemoveFile(filename_lip) if is_nxopus: util.RemoveFile(filename_nxopus) else: util.RemoveFile(filename_dsp0) if channel_count > 1: util.RemoveFile(filename_dsp1) if has_fuz and lip_size == 0: util.RemoveFile(filename_fuz) return True
def WriteIniFile(file, buffer): newSkyrimIniFile = target + "\\" + file util.LogInfo("Writing " + file) with open(newSkyrimIniFile, "w+") as f: f.write(buffer)
def LoadOrder(origin, target, loadOrderName): targetData = target + r"\Data" util.LogDebug("This is the origin: " + origin) util.LogDebug("This is the target: " + target) util.LogDebug("This is the target Data: " + targetData) util.LogDebug("This is the load order name: " + loadOrderName) util.LogInfo("Packed Load Order " + loadOrderName) loadOrderTxt = os.path.join(origin, loadOrderName + ".txt") loadOrder = open(loadOrderTxt, 'r').read() #util.LogDebug("LOAD ORDER <" + loadOrder + ">") loadOrderList = loadOrder.splitlines() loadOrderStart = int(loadOrderList[0]) loadOrderList = loadOrderList[1:] loadOrderList = list(filter(None, loadOrderList)) util.LogDebug("ESP Start at " + str(loadOrderStart)) util.LogDebug("LOAD ORDER LIST <" + str(loadOrderList) + ">") pristineFolder = target pristineSkyrimIni = pristineFolder + r"\Skyrim.ini" util.LogDebug("Attempt to open " + pristineSkyrimIni) pristineSkyrim = open(pristineSkyrimIni, 'r').read() #util.LogDebug("PRISTINE:\n" + pristineSkyrim + "END PRISTINE") newSkyrimIni = pristineSkyrim languageInis = {} for file in os.listdir(pristineFolder): if file.endswith(".ini") and file != "Skyrim.ini" and file != "desktop.ini": util.LogDebug("Language ini <" + file + ">") languageInis[file] = pristineFolder + "\\" + file util.LogDebug("Language Inis:") for languageIni in languageInis: liFilename = languageInis[languageIni] util.LogDebug(languageIni+ " " + liFilename) def GetArchiveList(aln, buffer, flags=None): util.LogDebug("Find existing " + aln) alPattern = "(" + aln + r"=[^\n$]*)[\n$]*" if flags != None: alPattern = re.compile(alPattern, flags) al = re.search(alPattern, buffer) if al == None: util.LogError("Cannot find " + aln + ", bailing") util.LogError("BUFFER" + buffer + "ENDBUFFER") sys.exit(1) retVal = al.group(1) util.LogDebug("<" + retVal + ">") return retVal sResourceArchiveList = GetArchiveList("sResourceArchiveList", newSkyrimIni) sResourceArchiveList2 = GetArchiveList("sResourceArchiveList2", newSkyrimIni) sArchiveToLoadInMemoryList = GetArchiveList("sArchiveToLoadInMemoryList", newSkyrimIni) def GetTestFile(tfn): tfn = str(tfn) util.LogDebug("Find test file " + tfn) tfPattern = "([;]*sTestFile" + tfn + r"=[^\n]*)\n" tf = re.search(tfPattern, newSkyrimIni) if tf == None: util.LogError("Cannot find " + tfn + ", bailing") sys.exit(1) retVal = tf.group(1) util.LogDebug("<" + retVal + ">") return retVal CurrentTestFileIDX = loadOrderStart sTestFiles = {} for i in range(CurrentTestFileIDX, 11): sTestFiles["sTestFile" + str(i)] = GetTestFile(i) def CopyFile(file, filename): newFileName = targetData + "\\" + file util.LogDebug(filename + "->" + newFileName) shutil.copy2(filename, newFileName) def InsertTestFile(name, filename): nonlocal CurrentTestFileIDX, newSkyrimIni currentTestFile = "sTestFile" + str(CurrentTestFileIDX) CurrentTestFileIDX = CurrentTestFileIDX + 1 newTestFile = currentTestFile + "=" + name newSkyrimIni = newSkyrimIni.replace(sTestFiles[currentTestFile], newTestFile) util.LogDebug(newTestFile) CopyFile(file, filename) #util.LogDebug("TESTFILES<" + str(sTestFiles) + ">") resourceArchiveList2Additions = '' newResourceArchiveList2 = sResourceArchiveList2 def InsertLanguageBSA(name, filename): nonlocal resourceArchiveList2Additions, newResourceArchiveList2 resourceArchiveList2Additions += ", " + name newResourceArchiveList2 += ", " + name util.LogDebug(newResourceArchiveList2) newResourceArchiveList = sResourceArchiveList def InsertMainBSA(name, filename): nonlocal newResourceArchiveList newResourceArchiveList += ", " + name util.LogDebug(newResourceArchiveList) newArchiveToLoadInMemoryList = sArchiveToLoadInMemoryList def InsertInMemoryBSA(name, filename): nonlocal newResourceArchiveList, newArchiveToLoadInMemoryList newResourceArchiveList += ", " + name newArchiveToLoadInMemoryList += ", " + name util.LogDebug(newResourceArchiveList) util.LogDebug(newArchiveToLoadInMemoryList) if loadOrderStart <= 4: util.LogDebug("Hyrule.esp will not be loaded, so removing Skyrim - Hyrule.bsa from newResourceArchiveList") newResourceArchiveList = newResourceArchiveList.replace(", Skyrim - Hyrule.bsa", "") util.LogDebug("newResourceArchiveList is now:\n" + newResourceArchiveList) iniPattern = r"^[; ]*([^=]*)=([^$]*)$" def InsertIni(filename): nonlocal newSkyrimIni buffer = open(filename, 'r').read() lines = buffer.splitlines() pendingAdditions = [] currentHeader = '' def EndHeader(): nonlocal newSkyrimIni if len(pendingAdditions) > 0 and currentHeader != '': currentHeaderSearch = currentHeader + "\n" util.LogDebug("Search for <" + currentHeaderSearch + ">") newHeader = currentHeaderSearch for newKeyValue in pendingAdditions: newHeader += newKeyValue newSkyrimIni = newSkyrimIni.replace(currentHeaderSearch, newHeader) for line in lines: util.LogDebug("INI <" + line + ">") if line.startswith("["): EndHeader() currentHeader = line pendingAdditions = [] else: newLine = line + "\n" myKeySearch = re.search(iniPattern, line) if myKeySearch == None: util.LogDebug("INI line no key/pair found") else: myKey = myKeySearch.group(1) myValue = myKeySearch.group(2) util.LogDebug("MyKey <" + myKey + ">") util.LogDebug("MyValue <" + myValue + ">") localIniPattern = re.compile(r"^[; ]*" + myKey + r"=([^\n\r ]*)[ \n\r]", re.MULTILINE) existingKeyValue = re.search(localIniPattern, newSkyrimIni) if existingKeyValue != None: wholePattern = existingKeyValue.group(0) value = existingKeyValue.group(1) util.LogDebug("***" + wholePattern + "->" + newLine + "***") newSkyrimIni = newSkyrimIni.replace(wholePattern, newLine) else: pendingAdditions.append(newLine) EndHeader() for plugin in loadOrderList: util.LogDebug("Found " + plugin) pluginFolder = os.path.join(origin, plugin) if plugin.startswith("#"): util.LogDebug("Passing on " + plugin) else: unpack_mod.UnpackMod(pluginFolder, targetData) for file in os.listdir(targetData): if file.endswith(".ini"): filename = os.path.join(targetData, file) os.remove(filename) for file in os.listdir(pluginFolder): util.LogDebug(" Found " + file) filename = os.path.join(pluginFolder, file) if file.endswith(".esm"): InsertTestFile(file, filename) elif file.endswith(".ini"): InsertIni(filename) elif file.endswith(".bsa"): pass # we unpacked already else: if os.path.isdir(filename): shutil.copytree(filename, os.path.join(targetData, file)) else: shutil.copy2(filename, os.path.join(targetData, file)) bsaList = pack_mod.PackMod(loadOrderName, targetData) for filename in bsaList: file = os.path.basename(filename) if file.endswith("Textures.bsa"): InsertLanguageBSA(file, filename) elif file.endswith("Voices.bsa"): InsertLanguageBSA(file, filename) elif file.endswith("Animations.bsa"): InsertInMemoryBSA(file, filename) elif file.endswith(".bsa"): InsertMainBSA(file, filename) def WriteIniFile(file, buffer): newSkyrimIniFile = target + "\\" + file util.LogInfo("Writing " + file) with open(newSkyrimIniFile, "w+") as f: f.write(buffer) newSkyrimIni = newSkyrimIni.replace(sResourceArchiveList, newResourceArchiveList) newSkyrimIni = newSkyrimIni.replace(sResourceArchiveList2, newResourceArchiveList2) newSkyrimIni = newSkyrimIni.replace(sArchiveToLoadInMemoryList, newArchiveToLoadInMemoryList) WriteIniFile("Skyrim.ini", newSkyrimIni) for languageIni in languageInis: liFilename = languageInis[languageIni] languageBuffer = open(liFilename, 'r').read() util.LogDebug("opening " + liFilename) liResourceArchiveList2 = GetArchiveList("sResourceArchiveList2", languageBuffer, re.MULTILINE) newLiResourceArchiveList2 = liResourceArchiveList2 + resourceArchiveList2Additions languageBuffer = languageBuffer.replace(liResourceArchiveList2, newLiResourceArchiveList2) WriteIniFile(languageIni, languageBuffer)
# Convert Audio if ok: ok = ConvertAudio(filename_wav, filepath_without_extension, is_nxopus, has_lip) return ok def ConvertSound(target, filepath_without_extension): return ConvertSound_Internal(filepath_without_extension) def ConvertSoundAsync(target, filename, logname, ret): util.InitialiseMPLog(logname) retVal = ConvertSound(target, filename) ret["retVal"] = retVal if __name__ == '__main__': import sys filepath = sys.argv[1] util.SetScriptPath(sys.argv[2]) util.LogInfo("Skyrim-NX-Toolkit {} - convert_sound_zappa".format( util.GetToolkitVersion())) retValue = ConvertSound_Internal(filepath) tempLog = util.GetTempLog() for msg in tempLog: print(msg) if not retValue: sys.exit(1) sys.exit(0)
def PackMod(mod_name, target): script_path = util.GetScriptPath() utilities_path = util.GetUtilitiesPath() bsarch = os.path.join(utilities_path, "bsarch.exe") util.LogDebug("This is the target: " + target) util.LogDebug("This is the mod name " + mod_name) util.LogInfo("Pack Mod") has_archive = util.HasArchive() util.LogDebug("HasArchive is {}".format(has_archive)) data_list = os.listdir(target) util.LogDebug(str(data_list)) BSARules = bsa_rules.GetBSARules() BSAs = {} SafePlugins = [ "skyrim.esm", "dawnguard.esm", "hearthfires.esm", "dragonborn.esm" ] PluginPaths = {} for plugin in SafePlugins: PluginPaths[plugin] = plugin # Arbitrary Limit to search for merged plugin list LineLimit = 10 ChunkSize = 1024 ChunksLimit = 10 childPattern = re.compile(r"Merged Plugin:([^\0]*)[\0]", re.MULTILINE) espPattern = re.compile(r"(.+?\.[^.]*$|$)", re.MULTILINE) def ReadPlugin(filename): util.LogDebug("Reading plugin{}".format(filename)) pluginList = [] with open(filename, "rb") as plugin: lineNumber = 0 chunkNumber = 0 foundMerge = False buffer = '' while True: chunk = plugin.read(ChunkSize) chunkNumber += 1 buffer = buffer + "".join(map(chr, chunk)) if "Merged Plugin:" in buffer: foundMerge = True if foundMerge: childPlugins = re.search(childPattern, buffer) if childPlugins != None: wholePattern = childPlugins.group(0) value = childPlugins.group(1) util.LogDebug("Found Plugins Block <{}>".format(value)) while True: espTest = re.findall(espPattern, value) if espTest != None: for espCandidate in espTest: espCandidate = espCandidate.strip( ) #''.join(espCandidate.split()) if espCandidate != '': util.LogDebug( "Found <{}>".format(espCandidate)) pluginList.append(espCandidate.lower()) #util.LogInfo("Found <{}>".format(str(espTest))) break break if chunkNumber >= ChunksLimit: break if not foundMerge: lineNumber += 1 if lineNumber >= LineLimit: break return pluginList mod_pathname = mod_name + ".esp" for root, subdirs, files in os.walk(target): for file in files: if file.endswith(".esp") or file.endswith(".esm"): filename = os.path.join(root, file) util.LogDebug("Found a plugin at {}, <{}>".format( filename, file)) # look after yourself PluginPaths[file.lower()] = file.lower() childrenOfPlugin = ReadPlugin(filename) if len(childrenOfPlugin) > 0: util.LogInfo( "Detected that {} is merged from:".format(file)) for child in childrenOfPlugin: util.LogInfo(" - {}".format(child)) # look after your children PluginPaths[child.lower()] = file.lower() mod_pathname = file # only interested in files in the root folder break util.LogDebug("PluginPaths is <{}>".format(str(PluginPaths))) def DefineBSA(bsa_name): nonlocal BSAs temp = os.path.join(target, "Temp") if bsa_name != '': temp = os.path.join(target, "Temp - " + bsa_name) temp_data = os.path.join(temp, "Data") util.RemoveTree(temp) os.makedirs(temp_data, exist_ok=True) BSAs[bsa_name] = temp_data numFoldersToMove = 0 numFilesToMove = 0 numFoldersMoved = 0 numFilesMoved = 0 def ApplyRuleToFolder(rule, folder): nonlocal numFoldersMoved bsa_name = rule["BSA"] if bsa_name not in BSAs: DefineBSA(bsa_name) move_to_folder = BSAs[bsa_name] child = os.path.basename(folder) parent = os.path.dirname(folder) if parent != target: util.LogDebug("parent is not target") rel_parent = os.path.relpath(parent, target) util.LogDebug("rel_parent is {}".format(rel_parent)) move_to_folder = os.path.join(move_to_folder, rel_parent, child) util.LogDebug("move_to_folder is {}".format(move_to_folder)) shutil.move(folder, move_to_folder) util.LogDebug("moving {} to {}".format(folder, move_to_folder)) numFoldersMoved += 1 sys.stdout.write("Moved Files {}/{} Folders {}/{} \r".format( numFilesMoved, numFilesToMove, numFoldersMoved, numFoldersToMove)) sys.stdout.flush() def ApplyRuleToFile(rule, file_path): nonlocal numFilesMoved bsa_name = rule["BSA"] if bsa_name not in BSAs: DefineBSA(bsa_name) relative_path = os.path.relpath(file_path, target) target_path = os.path.join(BSAs[bsa_name], relative_path) target_folder = os.path.dirname(target_path) os.makedirs(target_folder, exist_ok=True) util.LogDebug("moving {} to {}".format(file_path, target_path)) shutil.move(file_path, target_path) numFilesMoved += 1 sys.stdout.write("Moved Files {}/{} Folders {}/{} \r".format( numFilesMoved, numFilesToMove, numFoldersMoved, numFoldersToMove)) sys.stdout.flush() ignoreMovedFolders = [] for root, subdirs, files in os.walk(target): relative_folder = os.path.relpath(root, target).lower() if root != target: # Check if this folder is ignored already ignoreThisFolder = False for ignored in ignoreMovedFolders: if relative_folder.startswith(ignored): ignoreThisFolder = True if not ignoreThisFolder: # Now check if there is an unqualified path rule for this folder (like "scripts" are only placed in Misc) folderCount = 0 lastRuleMatch = None for rule in BSARules: if "Folder" in rule and relative_folder.startswith( rule["Folder"]): folderCount += 1 lastRuleMatch = rule if folderCount == 1: numFoldersToMove += 1 ignoreMovedFolders.append(relative_folder) else: for filename in files: util.LogDebug(os.path.join(relative_folder, filename)) numFilesToMove += 1 util.LogInfo("Files ({}) / Folders ({}) to move".format( numFilesToMove, numFoldersToMove)) RemoveFolders = [] for root, subdirs, files in os.walk(target): relative_folder = os.path.relpath(root, target).lower() #util.LogDebug("Walking relative folder " + relative_folder) if root == target: for child in subdirs: util.LogDebug("Children {}".format(child)) RemoveFolders.append(os.path.join(target, child)) else: # First check if there is an unqualified path rule for this folder (like "scripts" are only placed in Misc) folderCount = 0 lastRuleMatch = None for rule in BSARules: if "Folder" in rule and relative_folder.startswith( rule["Folder"]): folderCount += 1 lastRuleMatch = rule if folderCount == 1: util.LogDebug("ApplyRuleToFolder({}) -> {}".format( lastRuleMatch["BSA"], relative_folder)) ApplyRuleToFolder(lastRuleMatch, root) else: #util.LogDebug("No folder match, check files") for filename in files: filename = filename.lower() file_path = os.path.join(root, filename) relative_path = os.path.relpath(file_path, target) should_apply = False for rule in BSARules: should_apply = True if should_apply and "Folder" in rule: #util.LogDebug("checking folder rule {} vs relative_folder {}".format(rule["Folder"], relative_folder)) should_apply = should_apply and relative_folder.startswith( rule["Folder"]) if should_apply and "Extension" in rule: #util.LogDebug("checking extension rule {} vs filename {}".format(rule["Extension"], filename)) should_apply = should_apply and filename.endswith( rule["Extension"]) if should_apply: util.LogDebug("Applying BSA {} for {}".format( rule["BSA"], file_path)) ApplyRuleToFile(rule, file_path) break if not should_apply: util.LogWarn("Could not apply rule for <{}>".format( relative_path)) for rule in BSARules: should_apply = True if should_apply and "Folder" in rule: util.LogDebug( "checking folder rule {} vs relative_folder {}" .format(rule["Folder"], relative_folder)) should_apply = should_apply and relative_folder.startswith( rule["Folder"]) if should_apply and "Extension" in rule: util.LogDebug( "checking extension rule {} vs filename {}" .format(rule["Extension"], filename)) should_apply = should_apply and filename.endswith( rule["Extension"]) sys.stdout.write("\n") util.LogInfo("Cleanup old folders") for folder in RemoveFolders: util.LogDebug("Cleanup {}".format(folder)) util.RemoveTree(folder) def CleanPluginSpecificPaths(cleanup_directory): MoveFromTo = [] DeleteUnmatched = [] for root, subdirs, files in os.walk(cleanup_directory): directory = root.lower() dir_name = os.path.basename(directory) if (dir_name.endswith("esm") or dir_name.endswith("esp")) and dir_name not in SafePlugins: if dir_name in PluginPaths: target_pathname = PluginPaths[dir_name.lower()] new_path = os.path.join(os.path.dirname(directory), target_pathname) if not os.path.isdir(new_path): util.LogDebug( "Rename plugin directory {} to {}".format( directory, new_path)) os.rename(directory, new_path) else: util.LogDebug( "Move plugin files from directory {} to {}".format( directory, new_path)) MoveFromTo.append((directory, new_path)) else: util.LogWarn("Marking <{}> for deletion.".format(dir_name)) DeleteUnmatched.append(directory) for moveFromTo in MoveFromTo: (move_from, move_to) = moveFromTo util.LogDebug("Need to move from {} to {}".format( move_from, move_to)) for root, subdirs, files in os.walk(move_from): for file in files: file_path = os.path.join(root, file) relative_path = os.path.relpath(root, move_from) new_path = os.path.join(move_to, relative_path, file) util.LogDebug("Moving file from {} to {}".format( file_path, new_path)) new_directory = os.path.dirname(new_path) os.makedirs(new_directory, exist_ok=True) shutil.move(file_path, new_path) for deleteDirectory in DeleteUnmatched: util.LogDebug("Deleting <{}>".format(deleteDirectory)) util.RemoveTree(deleteDirectory) util.LogDebug("Deleted <{}>".format(deleteDirectory)) util.LogDebug("Build BSAs") bsaList = [] for bsa_name in BSAs: temp_data = BSAs[bsa_name] temp = os.path.dirname(temp_data) bsa_file_suffix = "" if bsa_name != "": bsa_file_suffix = " - " + bsa_name CleanPluginSpecificPaths(temp_data) bsa_filename = mod_name + bsa_file_suffix + ".bsa" target_bsa = os.path.join(target, bsa_filename) useArchive = has_archive if useArchive: bsa_list = archive_bsa.ArchiveBSA(temp, bsa_filename) for bsa_info in bsa_list: bsa_filename = bsa_info["FileName"] bsa_filepath = bsa_info["Folder"] bsa_fullpath = os.path.join(bsa_filepath, bsa_filename) newTargetBSA = os.path.join(target, bsa_filename) shutil.move(bsa_fullpath, newTargetBSA) bsaList.append(newTargetBSA) else: bsa_list = bsarch_bsa.BsarchBSA(temp, target_bsa) for bsa_info in bsa_list: bsa_filename = bsa_info["FileName"] bsa_filepath = bsa_info["Folder"] bsa_fullpath = os.path.join(bsa_filepath, bsa_filename) newTargetBSA = os.path.join(target, bsa_filename) shutil.move(bsa_fullpath, newTargetBSA) bsaList.append(newTargetBSA) util.RemoveTree(temp) util.LogDebug("PackMod Done") return bsaList
def ArchiveBSA(target_folder, bsa_filename): script_path = util.GetScriptPath() utilities_path = util.GetUtilitiesPath() archive_original = os.path.join(utilities_path, "Archive.exe") util.LogDebug("Copy Archive.exe to target folder") archive = os.path.join(target_folder, "Archive.exe") shutil.copy2(archive_original, archive) log_basename = "log.txt" log_filename = os.path.join(target_folder, log_basename) config_basename = "bsa_config.txt" config_filename = os.path.join(target_folder, config_basename) allFilesList = [] checks = {} util.LogInfo("Build File List") totalFileSizeTally = 0 target_data = os.path.join(target_folder, "Data") util.LogDebug("Walking the target directory " + target_data) for root, subdirs, files in os.walk(target_data): util.LogDebug('--\nroot = ' + root) if root == target_data: util.LogDebug("subdirs: " + str(subdirs)) lower_case_data_list = [x.lower() for x in subdirs] util.LogDebug("lcds: " + str(lower_case_data_list)) if "meshes" in lower_case_data_list: util.LogDebug("found meshes") checks["Retain Strings During Startup"] = True checks["Meshes"] = True if "textures" in lower_case_data_list: util.LogDebug("found texttures") checks["Textures"] = True if "interface" in lower_case_data_list: util.LogDebug("found interface") checks["Menus"] = True if "music" in lower_case_data_list: util.LogDebug("found music") checks["Retain File Names"] = True checks["Sounds"] = True if "sound" in lower_case_data_list: util.LogDebug("found sound") sound_list = os.listdir(os.path.join(target_data, "sound")) sound_list_lower = [x.lower() for x in sound_list] if "fx" in sound_list_lower: util.LogDebug("found sound//fx") checks["Retain File Names"] = True checks["Sounds"] = True if "voice" in sound_list_lower: util.LogDebug("found sound//voice") checks["Voices"] = True if "shadersfx" in lower_case_data_list: util.LogDebug("found shaders") checks["Shaders"] = True if "seq" in lower_case_data_list: util.LogDebug("found seq") checks["Retain File Names"] = True checks["Misc"] = True if "grass" in lower_case_data_list: util.LogDebug("found grass") checks["Retain File Names"] = True checks["Misc"] = True if "scripts" in lower_case_data_list: util.LogDebug("found scripts") checks["Retain File Names"] = True checks["Misc"] = True else: for filename in files: if filename != "desktop.ini": file_path = os.path.join(root, filename) relative_path = file_path.replace(target_folder, '') util.LogDebug('\t- file %s (relative path: %s)' % (filename, relative_path)) path_no_data = relative_path[6:] file_size = os.path.getsize(file_path) totalFileSizeTally += file_size util.LogDebug("totalFileSizeTally is now: " + str(totalFileSizeTally)) allFilesList.append({ 'FileName': filename, 'FilePath': file_path, 'RelativePath': relative_path, 'PathNoData': path_no_data, 'FileSize': file_size }) SizeLimitBSA = bsa_rules.BSASizeLimit currentFileIndex = None currentFileSizeTally = 0 buffer = '' bsaFileWritten = [] bsa_original_filename = bsa_filename def WrtiteBSA(): nonlocal currentFileIndex, buffer, bsa_filename util.LogDebug("Writing BSA with filelist:<" + buffer + ">") filelist_basename = "bsa_filelist.txt" if currentFileIndex != None: filelist_basename = "bsa_filelist" + str(currentFileIndex) + ".txt" bsa_filename = bsa_original_filename[:-4] + str( currentFileIndex) + ".bsa" currentFileIndex += 1 filelist_filename = os.path.join(target_folder, filelist_basename) with open(filelist_filename, 'w') as filelist_file: filelist_file.write(buffer) buffer = '' util.LogInfo("Build Config") checksOrder = [ "Meshes", "Textures", "Menus", "Sounds", "Voices", "Shaders", "Trees", "Fonts", "Misc", "Compress Archive", "Retain Directory Names", "Retain File Names", "Retain File Name Offsets", "Retain Strings During Startup", "XBox 360 Archive", "Embed File Names" ] with open(config_filename, 'w') as config_file: config_file.write("Log: " + log_basename + "\n") config_file.write("New Archive\n") for check in checksOrder: if check in checks: config_file.write("Check: " + check + "\n") config_file.write("Set File Group Root: Data\\\n") config_file.write("Add File Group: " + filelist_basename + "\n") config_file.write("Save Archive: " + bsa_filename + "\n") util.LogInfo("Run Archive.exe") commandLine = ["Archive.exe", config_basename] os.chdir(target_folder) util.RunCommandLine(commandLine) with open(log_filename, "r") as log_file: for line in log_file: util.LogDebug(line) os.remove(log_filename) os.remove(filelist_filename) os.remove(config_filename) bsaFileWritten.append({ "Folder": target_folder, "FileName": bsa_filename }) if totalFileSizeTally > SizeLimitBSA: currentFileIndex = 0 totalWrittenTally = 0 for fileInfo in allFilesList: file_size = fileInfo['FileSize'] newTally = currentFileSizeTally + file_size totalWrittenTally = totalWrittenTally + file_size util.LogDebug("Adding " + fileInfo['FileName'] + " currentFileSizeTally is " + str(currentFileSizeTally) + " file_size is " + str(file_size) + " totalWrittenTally is " + str(totalWrittenTally)) buffer += fileInfo['PathNoData'] + "\n" currentFileSizeTally += file_size if (newTally >= SizeLimitBSA) or (totalWrittenTally >= totalFileSizeTally): WrtiteBSA() currentFileSizeTally = 0 if buffer != '': util.LogWarn("BUFFER NOT EMPTY!") util.LogInfo("Clean Up") util.RemoveFile(archive) os.chdir(script_path) return bsaFileWritten
def ConvertSound_Internal(filepath_without_extension): """ Converts PC SSE sound files to be compatible with NX SSE supported codecs """ filename_wav = filepath_without_extension + ".wav" filename_xwm = filepath_without_extension + ".xwm" filename_lip = filepath_without_extension + ".lip" filename_fuz = filepath_without_extension + ".fuz" has_wav = os.path.exists(filename_wav) has_xwm = os.path.exists(filename_xwm) has_lip = os.path.exists(filename_lip) has_fuz = os.path.exists(filename_fuz) util.LogDebug( "INFO: Convert Sound <{}> WAV:{} XWM:{} LIP:{} FUZ:{}".format( filepath_without_extension, has_wav, has_xwm, has_lip, has_fuz)) # UNFUZ Audio if has_fuz: try: with open(filename_fuz, "rb") as fuz_file: fuz_file.seek(0x08) lip_size = int.from_bytes(fuz_file.read(0x04), byteorder='little', signed=False) lip_data = fuz_file.read(lip_size) audio_data = fuz_file.read() except: util.LogInfo( "ERROR: failed to open FUZ <{}>.".format(filename_lip)) return False # determine AUDIO format audio_format = audio_data[0x08:0x0C] if audio_format == b'WAVE': has_wav = True filename_audio = filename_wav elif audio_format == b'XWMA': has_xwm = True filename_audio = filename_xwm else: util.LogInfo("ERROR: unknown audio format {} on FUZ <{}>.".format( audio_format, filename_fuz)) return False # save LIP contents if lip_size > 0: try: with open(filename_lip, "wb") as lip_file: lip_file.write(lip_data) has_lip = True util.LogDebug( "INFO: LIP created on disk from FUZ {}.".format( filename_fuz)) except: util.LogDebug( "ERROR: failed to create intermediate LIP <{}>.".format( filename_lip)) return False # save AUDIO contents try: with open(filename_audio, "wb") as audio_file: audio_file.write(audio_data) util.LogDebug( "INFO: AUDIO created on disk from FUZ {}.".format( filename_fuz)) except: util.LogDebug( "ERROR: failed to create intermediate AUDIO <{}>.".format( filename_audio)) return False # get rid of the source PC FUZ file util.RemoveFile(filename_fuz) elif has_xwm: filename_audio = filename_xwm elif has_wav: filename_audio = filename_wav else: util.LogDebug("PANIC: IT SHOULD NEVER REACH THIS BRANCH...") return False # Force anything VOICE to use OPUS codec is_nxopus = "\\sound\\voice\\" in filepath_without_extension.lower() # Normalize Audio ok = NormalizeAudio(filename_audio, filepath_without_extension, is_nxopus) # Convert Audio if ok: ok = ConvertAudio(filename_wav, filepath_without_extension, is_nxopus, has_lip) return ok