def PackMod_External(mod_name, target):
util.InitialiseLog(target + ".log")
util.StartTimer()
util.LogInfo("Skyrim-NX-Toolkit {} - pack_mod".format(
util.GetToolkitVersion()))
PackMod(mod_name, target)
util.EndTimer()
def LoadOrder_External(origin, target, loadOrderName):
util.InitialiseLog(os.path.join(origin, loadOrderName) + ".log")
util.StartTimer()
util.LogInfo("Skyrim-NX-Toolkit {} - load_order".format(
util.GetToolkitVersion()))
LoadOrder(origin, target, loadOrderName)
util.EndTimer()
def RepackMod_External(origin, target):
util.InitialiseLog(origin + ".log")
util.StartTimer()
util.LogInfo("Skyrim-NX-Toolkit {} - repack_mod".format(
util.GetToolkitVersion()))
RepackMod(origin, target)
util.EndTimer()
def WriteBSA():
nonlocal currentFileIndex, bsa_filename
util.LogDebug("Writing BSA")
if currentFileIndex != None:
bsa_filename = bsa_original_filename[:-4] + str(
currentFileIndex) + ".bsa"
currentFileIndex += 1
util.LogInfo("Run bsarch.exe")
flags_value = flags.GetValue()
flags_hexvalue = hex(flags_value)
compress = "-z" if flags.IsSet(Flag_Compressed) else ""
commandLine = [
bsarch, "pack", temp_data, bsa_filename, "-sse", compress,
"-af:" + flags_hexvalue
]
util.RunCommandLine(commandLine)
bsaFileWritten.append({
"Folder": target_folder,
"FileName": bsa_filename
})
util.RemoveTree(temp_data)
def NXOPUS2FUZ(filename_nxopus, channel_count, sound_duration_ms, sound_file):
""" adds required NX SSE header to a nxopus sound """
try:
with open(filename_nxopus, "rb") as nxopus_file:
nxopus_data = bytearray(nxopus_file.read())
nxopus_size = len(nxopus_data)
except:
util.LogInfo("ERROR: failed to open intermediate NXOPUS <{}>.".format(
filename_nxopus))
return True
nx_sse_opus_header_size = b'\x14\x00\x00\x00'
nx_sse_opus_signature = b'\x0A\x8D\xD5\xFF'
sound_file.write(nx_sse_opus_signature)
sound_file.write(
sound_duration_ms.to_bytes(4, byteorder='little', signed=False))
sound_file.write(
channel_count.to_bytes(4, byteorder='little', signed=False))
sound_file.write(nx_sse_opus_header_size)
sound_file.write(nxopus_size.to_bytes(4, byteorder='little', signed=False))
sound_file.write(nxopus_data)
return False
def ConvertPath_External(mod_name, target):
util.InitialiseLog(target + ".log")
util.StartTimer()
util.LogInfo("Skyrim-NX-Toolkit {} - convert_path".format(
util.GetToolkitVersion()))
ConvertPath(mod_name, target)
util.EndTimer()
def ConvertHKX64_Internal(filename):
util.LogDebug("ConvertHKX64 : " + filename)
with open(filename, "rb") as hkx64:
payload = hkx64.read()
header_location = payload.find(b'\x00' * 15 + b'\x80' + b'\x00' * 48)
is_skeleton_container = payload.find(b'hkaSkeleton') > 0
is_behavior_container = payload.find(b'hkbBehavior') > 0
# it should not happen here as convert_path already doing some checks
if is_behavior_container or is_skeleton_container:
util.LogInfo("Warning, cannot convert {} because <{}>".format(
filename, 'SSE hkx animation'))
return False
# swap some fields in the AMD64 HKX header structure to make it PS4 compatible
if header_location > 0:
is_project_data = payload.find(b'hkbProjectData')
offset = 72 if is_project_data > 0 else 76
with open(filename, "wb") as hkxPS4:
hkxPS4.write(payload[0:18])
hkxPS4.write(b'\x01')
hkxPS4.write(payload[19:header_location + 16])
hkxPS4.write(payload[header_location + 20:header_location +
offset])
hkxPS4.write(payload[header_location:header_location + 4])
hkxPS4.write(payload[header_location + offset:])
return True
def ConvertMod_External(origin, target, oldrim):
print("<{}>".format(origin))
util.InitialiseLog(origin + ".log")
util.StartTimer()
util.LogInfo("Skyrim-NX-Toolkit {} - convert_mod".format(util.GetToolkitVersion()))
ConvertMod(origin, target, oldrim)
util.EndTimer()
def WrtiteBSA():
nonlocal currentFileIndex, buffer, bsa_filename
util.LogDebug("Writing BSA with filelist:<" + buffer + ">")
filelist_basename = "bsa_filelist.txt"
if currentFileIndex != None:
filelist_basename = "bsa_filelist" + str(currentFileIndex) + ".txt"
bsa_filename = bsa_original_filename[:-4] + str(
currentFileIndex) + ".bsa"
currentFileIndex += 1
filelist_filename = os.path.join(target_folder, filelist_basename)
with open(filelist_filename, 'w') as filelist_file:
filelist_file.write(buffer)
buffer = ''
util.LogInfo("Build Config")
checksOrder = [
"Meshes", "Textures", "Menus", "Sounds", "Voices", "Shaders",
"Trees", "Fonts", "Misc", "Compress Archive",
"Retain Directory Names", "Retain File Names",
"Retain File Name Offsets", "Retain Strings During Startup",
"XBox 360 Archive", "Embed File Names"
]
with open(config_filename, 'w') as config_file:
config_file.write("Log: " + log_basename + "\n")
config_file.write("New Archive\n")
for check in checksOrder:
if check in checks:
config_file.write("Check: " + check + "\n")
config_file.write("Set File Group Root: Data\\\n")
config_file.write("Add File Group: " + filelist_basename + "\n")
config_file.write("Save Archive: " + bsa_filename + "\n")
util.LogInfo("Run Archive.exe")
commandLine = ["Archive.exe", config_basename]
os.chdir(target_folder)
util.RunCommandLine(commandLine)
with open(log_filename, "r") as log_file:
for line in log_file:
util.LogDebug(line)
os.remove(log_filename)
os.remove(filelist_filename)
os.remove(config_filename)
bsaFileWritten.append({
"Folder": target_folder,
"FileName": bsa_filename
})
def DSP2MCADPCM(filename_dsp0, filename_dsp1, channel_count, sound_file):
""" creates a MCADPCM file from DSPADPCM ones """
try:
with open(filename_dsp0, "rb") as dsp0_file:
dsp0_data = bytearray(dsp0_file.read())
dsp0_size = len(dsp0_data)
DSP2LITTLE_ENDIAN(dsp0_data, 0x00)
except:
util.LogInfo("ERROR: failed to open intermediate DSP <{}>.".format(
filename_dsp0))
return True
if channel_count > 1:
try:
with open(filename_dsp1, "rb") as dsp1_file:
dsp1_data = bytearray(dsp1_file.read())
dsp1_size = len(dsp1_data)
dsp1_offset = 0x14 + dsp0_size
DSP2LITTLE_ENDIAN(dsp1_data, 0x00)
except:
util.LogInfo("ERROR: failed to open intermediate DSP <{}>.".format(
filename_dsp1))
return True
header_stereo = b'\x02\x00\x00\x00\x14\x00\x00\x00'
sound_file.write(header_stereo)
sound_file.write(
dsp0_size.to_bytes(4, byteorder='little', signed=False))
sound_file.write(
dsp1_offset.to_bytes(4, byteorder='little', signed=False))
sound_file.write(
dsp1_size.to_bytes(4, byteorder='little', signed=False))
sound_file.write(dsp0_data)
sound_file.write(dsp1_data)
else:
header_single = b'\x01\x00\x00\x00\x0C\x00\x00\x00'
sound_file.write(header_single)
sound_file.write(
dsp0_size.to_bytes(4, byteorder='little', signed=False))
sound_file.write(dsp0_data)
return False
def ConvertTXT_Internal(filename):
buffer = None
with open(filename, "rb") as pcFile:
buffer = pcFile.read()
buffer = bytearray(buffer)
util.LogInfo("Buffer0<{}>".format(buffer[0]))
util.LogInfo("Buffer0<{}>".format(buffer[1]))
i = 1
while i < len(buffer):
buffer[i:i] = b'\x00\x00\x00'
i += 4
util.LogDebug("BUFFER<{}>".format(buffer))
with open(filename, "wb") as outFile:
outFile.write(b'\xff\xfe\x00\x00')
outFile.write(buffer)
return True
def ConvertMod(origin, target, oldrim):
mod_name = os.path.basename(origin)
'''
util.LogDebug("This is the origin: " + origin)
util.LogDebug("This is the target: " + target)
util.LogDebug("This is the mod name " + mod_name)
'''
util.LogDebug("convert_mod.py 2.0")
toolkit_path = util.GetToolKitPath()
util.LogInfo("Convert Mod, ToolkitPath is {}".format(toolkit_path))
util.LogInfo("Convert Mod, create empty folder at target")
util.CreateTarget(target)
unpack_mod.UnpackMod(origin, target)
if oldrim:
reconcile_hkx.ReconcileHKX(target, oldrim)
convert_path.ConvertPath(mod_name, target)
pack_mod.PackMod(mod_name, target)
def ReconcileHKX(mod_path, oldrim_path):
script_path = util.GetScriptPath()
util.LogInfo("Reconcile HKX")
util.LogDebug("This is the mod_path: " + mod_path)
util.LogDebug("This is the oldrim_path " + oldrim_path)
CopyHKX = []
totalCount = 0
matchedCount = 0
for root, subdirs, files in os.walk(mod_path):
util.LogDebug("Walking folder " + root)
for filename in files:
if filename.lower().endswith(".hkx"):
file_path = os.path.join(root, filename)
relative_path = os.path.relpath(file_path, mod_path)
util.LogDebug("Relative path {} OR Path {}".format(
relative_path, oldrim_path))
oldrim_file_path = os.path.join(oldrim_path, relative_path)
totalCount += 1
util.LogDebug("Found {}, checking {}".format(
file_path, oldrim_file_path))
if os.path.exists(oldrim_file_path):
util.LogDebug(
"Found {} match in oldrim".format(oldrim_file_path))
matchedCount += 1
CopyHKX.append((file_path, oldrim_file_path))
util.LogInfo("Matched {}/{} hkx files in the mod".format(
matchedCount, totalCount))
for i in range(len(CopyHKX)):
(copy_to, copy_from) = CopyHKX[i]
util.LogDebug("Copying {}->{}".format(copy_from, copy_to))
shutil.copy2(copy_from, copy_to)
sys.stdout.write("Reconciled {}/{} \r".format(i + 1, len(CopyHKX)))
sys.stdout.flush()
sys.stdout.write("\n")
def get_setting(section, setting):
"""
Print out a setting
"""
config = get_config()
try:
value = config.get(section, setting)
except:
value = DefaultValues[section][setting]
util.LogInfo("Added default setting {}/{} as {}".format(
section, setting, value))
update_setting(section, setting, str(value))
return value
def RepackMod(origin, target):
mod_name = os.path.basename(origin)
'''
util.LogDebug("This is the origin: " + origin)
util.LogDebug("This is the target: " + target)
util.LogDebug("This is the mod name " + mod_name)
'''
util.LogDebug("convert_mod.py 2.0")
util.LogInfo("Convert Mod, create empty folder at target")
util.CreateTarget(target)
unpack_mod.UnpackMod(origin, target)
pack_mod.PackMod(mod_name, target)
def check_clear():
config = get_config()
util.LogInfo("Checking if settings.ini should be updated.")
clearOldConfig = False
currentVersion = util.GetToolkitVersion()
try:
settingsVersion = config.get("Version", "ToolkitVersion")
util.LogInfo(
"Version:ToolkitVersion is {}. Currrent Version is {}".format(
settingsVersion, currentVersion))
if settingsVersion != currentVersion:
util.LogInfo("Not a match. Clearing settings.ini")
clearOldConfig = True
except:
util.LogInfo(
"Version:ToolkitVersion was not found. Clearing settings.ini")
clearOldConfig = True
if clearOldConfig:
util.LogInfo("Clearing settings.ini")
toolkit_path = util.GetToolKitPath()
path = os.path.join(toolkit_path, "settings.ini")
create_config(path)
config.read(path)
while i < len(buffer):
buffer[i:i] = b'\x00\x00\x00'
i += 4
util.LogDebug("BUFFER<{}>".format(buffer))
with open(filename, "wb") as outFile:
outFile.write(b'\xff\xfe\x00\x00')
outFile.write(buffer)
return True
def ConvertTXT(target, filename):
return ConvertTXT_Internal(filename)
def ConvertTXTAsync(target, filename, logname, ret):
util.InitialiseMPLog(logname)
retVal = ConvertTXT(target, filename)
ret["retVal"] = retVal
if __name__ == '__main__':
filename = sys.argv[1]
util.InitialiseLog(filename + ".log")
util.StartTimer()
util.LogInfo("Skyrim-NX-Toolkit {} - convert_txt".format(
util.GetToolkitVersion()))
ConvertTXT_Internal(filename)
util.EndTimer()
def BsarchBSA(target_folder, bsa_filename):
script_path = util.GetScriptPath()
utilities_path = util.GetUtilitiesPath()
bsarch = os.path.join(utilities_path, "bsarch.exe")
log_basename = "log.txt"
log_filename = os.path.join(target_folder, log_basename)
config_basename = "bsa_config.txt"
config_filename = os.path.join(target_folder, config_basename)
allFilesList = []
Flag_NamedDir = 1
Flag_NamedFiles = 2
Flag_Compressed = 4
Flag_RetainDir = 8
Flag_RetainName = 16
Flag_RetainFOff = 32
Flag_XBox360 = 64
Flag_StartupStr = 128
Flag_EmbedName = 256
Flag_XMem = 512
Flag_Bit = 1024
flags = bitflag.BitFlag()
flags.SetFlag(Flag_NamedDir)
flags.SetFlag(Flag_NamedFiles)
util.LogInfo("Build File List")
totalFileSizeTally = 0
target_data = os.path.join(target_folder, "Data")
util.LogDebug("Walking the target directory " + target_data)
bsaFolders = []
SizeLimitBSA = bsa_rules.BSASizeLimit
totalFileCount = 0
for root, subdirs, files in os.walk(target_data):
util.LogDebug('--\nroot = ' + root)
if root != target_data:
for filename in files:
if filename != "desktop.ini" and filename != 'thumbs.db':
file_path = os.path.join(root, filename)
file_size = os.path.getsize(file_path)
totalFileSizeTally += file_size
totalFileCount += 1
currentFileIndex = None
if totalFileSizeTally > SizeLimitBSA:
currentFileIndex = 0
totalWrittenTally = 0
currentFileSizeTally = 0
buffer = ''
bsaFileWritten = []
bsa_original_filename = bsa_filename
temp_data = os.path.join(os.path.dirname(target_data), "Ready")
def WriteBSA():
nonlocal currentFileIndex, bsa_filename
util.LogDebug("Writing BSA")
if currentFileIndex != None:
bsa_filename = bsa_original_filename[:-4] + str(
currentFileIndex) + ".bsa"
currentFileIndex += 1
util.LogInfo("Run bsarch.exe")
flags_value = flags.GetValue()
flags_hexvalue = hex(flags_value)
compress = "-z" if flags.IsSet(Flag_Compressed) else ""
commandLine = [
bsarch, "pack", temp_data, bsa_filename, "-sse", compress,
"-af:" + flags_hexvalue
]
util.RunCommandLine(commandLine)
bsaFileWritten.append({
"Folder": target_folder,
"FileName": bsa_filename
})
util.RemoveTree(temp_data)
filesArchived = 0
for root, subdirs, files in os.walk(target_data):
util.LogDebug('--\nroot = ' + root)
if root == target_data:
util.LogDebug("subdirs: " + str(subdirs))
lower_case_data_list = [x.lower() for x in subdirs]
util.LogDebug("lcds: " + str(lower_case_data_list))
if "meshes" in lower_case_data_list:
util.LogDebug("found meshes")
flags.SetFlag(Flag_StartupStr)
flags.SetFlag(Flag_Compressed)
if "textures" in lower_case_data_list:
util.LogDebug("found textures")
if "interface" in lower_case_data_list:
util.LogDebug("found interface")
if "music" in lower_case_data_list:
util.LogDebug("found music")
flags.SetFlag(Flag_RetainName)
if "sound" in lower_case_data_list:
util.LogDebug("found sound")
sound_list = os.listdir(os.path.join(target_data, "sound"))
sound_list_lower = [x.lower() for x in sound_list]
if "fx" in sound_list_lower:
util.LogDebug("found sound//fx")
flags.SetFlag(Flag_RetainName)
if "voice" in sound_list_lower:
util.LogDebug("found sound//voice")
if "shadersfx" in lower_case_data_list:
util.LogDebug("found shaders")
if "seq" in lower_case_data_list:
util.LogDebug("found seq")
flags.SetFlag(Flag_RetainName)
if "grass" in lower_case_data_list:
util.LogDebug("found grass")
flags.SetFlag(Flag_RetainName)
if "scripts" in lower_case_data_list:
util.LogDebug("found scripts")
flags.SetFlag(Flag_RetainName)
else:
for filename in files:
if filename != "desktop.ini" and filename != 'thumbs.db':
file_path = os.path.join(root, filename)
file_size = os.path.getsize(file_path)
newTally = currentFileSizeTally + file_size
util.LogDebug("Attempting to add " + file_path +
" currentFileSizeTally is " +
str(currentFileSizeTally) +
" file_size is " + str(file_size))
if (newTally >= SizeLimitBSA):
util.LogDebug(
"New BSA would be too big, writing current BSA")
sys.stdout.write("\n")
WriteBSA()
currentFileSizeTally = 0
relative_path = file_path.replace(target_folder, '')
path_no_data = relative_path[6:]
temp_path = os.path.join(temp_data, path_no_data)
util.LogDebug("Moving <{}> to <{}>".format(
file_path, temp_path))
paths_to_create = []
check_path = os.path.dirname(temp_path)
util.LogDebug("Checking path {}".format(check_path))
while not os.path.isdir(check_path):
util.LogDebug("{} does not exist.".format(check_path))
paths_to_create.insert(0, check_path)
check_path = os.path.dirname(check_path)
util.LogDebug("Checking path {}".format(check_path))
for path_to_create in paths_to_create:
util.LogDebug("{} does not exist.".format(check_path))
os.mkdir(path_to_create)
shutil.move(file_path, temp_path)
currentFileSizeTally += file_size
filesArchived += 1
sys.stdout.write("Prepared {}/{} \r".format(
filesArchived, totalFileCount))
sys.stdout.flush()
sys.stdout.write("\n")
if currentFileSizeTally > 0:
WriteBSA()
util.LogInfo("Clean Up")
os.chdir(script_path)
return bsaFileWritten
def ConvertPath(mod_name, target):
script_path = util.GetScriptPath()
util.LogInfo("Convert Path")
util.LogDebug("This is the target: " + target)
util.LogDebug("This is the mod name " + mod_name)
has_havoc = util.HasHavokBPP()
do_meshes = \
toolkit_config.get_bool_setting("Meshes", "RemoveEditorMarker") or \
toolkit_config.get_bool_setting("Meshes", "PrettySortBlocks") or \
toolkit_config.get_bool_setting("Meshes", "TrimTexturesPath") or \
toolkit_config.get_bool_setting("Meshes", "OptimizeForSSE")
NoConversion = {}
WarnConversion = {}
ConvertListDDS = []
ConvertListHKX = []
ConvertListHKX64 = []
ConvertListTXT = []
ConvertListSound = []
ConvertListMesh = []
for root, subdirs, files in os.walk(target):
if root != target:
util.LogDebug("Walking folder " + root)
for filename in files:
#util.LogDebug("filename: {}".format(filename))
if filename.lower().endswith(".dds"):
file_path = os.path.join(root, filename)
ddsChecked = check_dds.CheckDDS(file_path, file_path)
if ddsChecked == check_dds.PC:
ConvertListDDS.append(file_path)
elif ddsChecked == check_dds.NX:
if 'DDS' not in NoConversion:
NoConversion['DDS'] = 0
NoConversion['DDS'] += 1
else:
if 'DDS' not in WarnConversion:
WarnConversion['DDS'] = []
WarnConversion['DDS'].append(
(filename, "Unknown DDS format"))
elif filename.lower().endswith(".hkx"):
file_path = os.path.join(root, filename)
hkxChecked = check_hkx.CheckHKX(file_path, file_path)
if hkxChecked == check_hkx.PC_32:
ConvertListHKX.append(file_path)
elif hkxChecked == check_hkx.PC_XML:
ConvertListHKX.append(file_path)
elif hkxChecked == check_hkx.PC_64:
# if root.lower().find("behaviors") > 0 or filename.lower().find("skeleton") > 0:
# if 'HKX' not in WarnConversion:
# WarnConversion['HKX'] = []
# WarnConversion['HKX'].append( (filename, "SSE hkx animation") )
# else:
ConvertListHKX64.append(file_path)
elif hkxChecked == check_hkx.NX_64:
if 'HKX' not in NoConversion:
NoConversion['HKX'] = 0
NoConversion['HKX'] += 1
else:
if 'HKX' not in WarnConversion:
WarnConversion['HKX'] = []
WarnConversion['HKX'].append(
(filename, "Unknown hkx animation format"))
elif filename.lower().startswith(
"translate_") and filename.lower().endswith(".txt"):
file_path = os.path.join(root, filename)
ConvertListTXT.append(file_path)
elif filename.lower().endswith(".xwm") or filename.lower(
).endswith(".fuz") or filename.lower().endswith(".wav"):
file_path = os.path.join(root, filename[:-4])
if not file_path in ConvertListSound:
ConvertListSound.append(file_path)
elif filename.lower().endswith(".nif"):
file_path = os.path.join(root, filename)
ConvertListMesh.append(file_path)
for fileType in NoConversion:
util.LogInfo("Found {} {} files that are already in NX format".format(
NoConversion[fileType], fileType))
for fileType in WarnConversion:
fileTypeWarnings = WarnConversion[fileType]
for i in range(len(fileTypeWarnings)):
util.LogInfo("Warning, cannot convert {} because <{}>".format(
fileTypeWarnings[i][0], fileTypeWarnings[i][1]))
util.LogInfo("Found {} dds files to convert".format(len(ConvertListDDS)))
if has_havoc:
util.LogInfo("Found {} 32-bit hkx files to convert".format(
len(ConvertListHKX)))
else:
util.LogInfo(
"Found {} 32-bit hkx files that won't convert as Havoc utility wasn't found."
.format(len(ConvertListHKX)))
util.LogInfo("Found {} 64-bit hkx files to convert".format(
len(ConvertListHKX64)))
util.LogInfo("Found {} txt files to convert".format(len(ConvertListTXT)))
util.LogInfo("Found {} sound files to convert".format(
len(ConvertListSound)))
if do_meshes:
util.LogInfo("Found {} mesh files to convert".format(
len(ConvertListMesh)))
else:
util.LogInfo("Found {} mesh files but won't touch them.".format(
len(ConvertListMesh)))
'''
def LogProgress(convertList, convertFn, name):
if len(convertList) > 0:
failedCount = 0
for i in range(len(convertList)):
file_path = convertList[i]
success = convertFn(target, file_path)
if not success:
failedCount += 1
sys.stdout.write("Converted {}/{} {} ({}) failed. \r".format(i+1, len(convertList), name, failedCount))
sys.stdout.flush()
sys.stdout.write("\n")
'''
def LogProgress(convertList, fnName, convertFn, name, threadSetting):
if len(convertList) > 0:
failedCount = 0
maxThreads = toolkit_config.get_int_setting(
"Performance", threadSetting)
jm = job_manager.JobManager(maxThreads)
convertedCount = 0
processedCount = 0
totalCount = len(convertList)
def cb(success):
nonlocal processedCount, convertedCount, failedCount
processedCount += 1
if success:
convertedCount += 1
else:
failedCount += 1
sys.stdout.write(
"{} Processed {}/{} ({}/{}) success/failure. \r".format(
name, processedCount, totalCount, convertedCount,
failedCount))
sys.stdout.flush()
for i in range(len(convertList)):
file_path = convertList[i]
job = job_manager.Job(cb, fnName, convertFn, target.lower(),
file_path.lower())
jm.AddJob(job)
jm.ProcessBatch()
sys.stdout.write(
"{} Processing Complete: {}/{} ({}/{}) success/failure. \r".
format(name, processedCount, totalCount, convertedCount,
failedCount))
sys.stdout.write("\n")
if processedCount != totalCount:
sys.stdout.write("Not all were processed.\n")
sys.stdout.flush()
LogProgress(ConvertListDDS, "ConvertDDS", convert_dds.ConvertDDS, "DDS",
"MaxTextureThreads")
if has_havoc:
LogProgress(ConvertListHKX, "ConvertHKX", convert_hkx.ConvertHKX,
"HKX 32-bit", "MaxAnimationThreads")
LogProgress(ConvertListHKX64, "ConvertHKX64", convert_hkx64.ConvertHKX64,
"HKX 64-bit", "MaxAnimationThreads")
LogProgress(ConvertListTXT, "ConvertTXT", convert_txt.ConvertTXT, "TXT",
"MaxOtherThreads")
LogProgress(ConvertListSound, "ConvertSound", convert_sound.ConvertSound,
"Sounds", "MaxSoundThreads")
if do_meshes:
LogProgress(ConvertListMesh, "ConvertMesh", convert_nif.ConvertNIF,
"Meshes", "MaxMeshThreads")
def WAV2PCM16WAV(filename_xwm, filename_wav, isNxOpus):
""" Normalizes the WAV file to be a proper PCM16 with correct sample rate for VGAudioCli """
try:
with open(filename_wav, "rb") as wav_file:
wav_header = wav_file.read(0xFF)
wav_format = wav_header[0x08:0x0C].decode()
except:
util.LogInfo("ERROR: failed to create intermediate WAV <{}>.".format(
filename_wav))
return (True, None, None)
# check for a collateral case where Mod Authors save XWM with WAV extension
if wav_format == "XWMA":
util.LogInfo(
"WARNING: <{}> has WAV extension but is a XWMA. Fixing it.".format(
filename_wav))
filename_temp = filename_wav + "temp.wav"
util.RemoveFile(filename_temp)
util.RenameFile(filename_wav, filename_temp)
xWMAEncode = GetxWMAEncode()
commandLine = [xWMAEncode, filename_temp, filename_wav]
util.RunCommandLine(commandLine)
util.RemoveFile(filename_temp)
try:
with open(filename_wav, "rb") as wav_file:
wav_header = wav_file.read(0xFF)
wav_format = wav_header[0x08:0x0C].decode()
except:
util.LogInfo(
"ERROR: failed to fix collateral case on WAV <{}>.".format(
filename_wav))
return (True, None, None)
wav_audio_format = int.from_bytes(wav_header[0x14:0x16],
byteorder='little',
signed=False)
wav_sample_rate = int.from_bytes(wav_header[0x18:0x1C],
byteorder='little',
signed=False)
wav_bits_per_sample = int.from_bytes(wav_header[0x22:0x24],
byteorder='little',
signed=False)
#
# make the WAV file compatible with VGAudioCLi
#
# OPUS CODEC requires 24000 or 48000 sample rates from a PCM16 stream
# DSPADPCM CODEC requires 22050 or 44100 sample rates from a PCM16 stream
#
# get the closest ressampling RATE
DESIRED_RATES = [24000, 48000] if isNxOpus else [22050, 44100]
try:
i = 0
while wav_sample_rate >= DESIRED_RATES[i]:
i += 1
except:
i = 1
override_sample_rate = str(DESIRED_RATES[i])
# ressample if required
if (isNxOpus and not wav_sample_rate in DESIRED_RATES
) or wav_audio_format != 1 or wav_bits_per_sample != 16:
filename_temp = filename_wav + "temp.wav"
util.RemoveFile(filename_temp)
util.RenameFile(filename_wav, filename_temp)
SndFileConvert = GetSndFileConvert()
commandLine = [
SndFileConvert, "-override-sample-rate=" + override_sample_rate,
"-pcm16", filename_temp, filename_wav
]
util.RunCommandLine(commandLine)
util.RemoveFile(filename_temp)
# really need to open one last time to get the metadata right
try:
with open(filename_wav, "rb") as wav_file:
wav_header = wav_file.read(0xFF)
wav_sample_rate = int.from_bytes(wav_header[0x18:0x1C],
byteorder='little',
signed=False)
wav_bits_per_sample = int.from_bytes(wav_header[0x22:0x24],
byteorder='little',
signed=False)
except:
util.LogInfo("ERROR: failed to create normalized WAV <{}>.".format(
filename_wav))
return (True, None, None)
wav_channel_count = int.from_bytes(wav_header[0x16:0x18],
byteorder='little',
signed=False)
try:
data_size_offset = wav_header.find(b'\x64\x61\x74\x61') + 0x04
wav_data_size = int.from_bytes(
wav_header[data_size_offset:data_size_offset + 0x04],
byteorder='little',
signed=False)
wav_duration_ms = int(
float(wav_data_size * 1000) / float(
wav_sample_rate * wav_channel_count * wav_bits_per_sample / 8))
except:
wav_duration_ms = 0
util.LogDebug(
"Converted WAV <{}>:\n DURATION:{} FORMAT:{} CHANNELS:{} SAMPLE_RATE:{} BITS_PER_SAMPLE:{} DATA_SIZE:{}"
.format(filename_wav, wav_duration_ms, wav_audio_format,
wav_channel_count, wav_sample_rate, wav_bits_per_sample,
wav_data_size))
return (False, wav_channel_count, wav_duration_ms)
def ConvertSound_Internal(filepath_without_extension):
""" Converts PC SSE sound files to be compatible with NX SSE supported codecs """
filename_mcadpcm = filepath_without_extension + ".mcadpcm"
filename_nxopus = filepath_without_extension + ".lopus"
filename_dsp0 = filepath_without_extension + "_CH0_.dsp"
filename_dsp1 = filepath_without_extension + "_CH1_.dsp"
filename_wav = filepath_without_extension + ".wav"
filename_xwm = filepath_without_extension + ".xwm"
filename_lip = filepath_without_extension + ".lip"
filename_fuz = filepath_without_extension + ".fuz"
has_wav = os.path.exists(filename_wav)
has_xwm = os.path.exists(filename_xwm)
has_lip = os.path.exists(filename_lip)
has_fuz = os.path.exists(filename_fuz)
# get the desired CODEC
if "\\music\\" in filepath_without_extension.lower():
codec = toolkit_config.get_setting("Sounds", "music")
elif "\\sound\\fx\\" in filepath_without_extension.lower():
codec = toolkit_config.get_setting("Sounds", "fx")
elif "\\sound\\voice\\" in filepath_without_extension.lower():
codec = toolkit_config.get_setting("Sounds", "voice")
else:
codec = "nxopus"
is_nxopus = codec.lower() == "nxopus"
util.LogDebug(
"Convert Sound <{}> WAV:{} XWM:{} LIP:{} FUZ:{} CODEC:{}".format(
filepath_without_extension, has_wav, has_xwm, has_lip, has_fuz,
codec))
# FUZ files always have precedence over loose WAV, XWM or LIP
lip_size = 0
if has_fuz:
try:
with open(filename_fuz, "rb") as fuz_file:
fuz_file.seek(0x08)
lip_size = int.from_bytes(fuz_file.read(0x04),
byteorder='little',
signed=False)
lip_data = fuz_file.read(lip_size)
if lip_size == 0:
util.LogDebug(
"INFO: FUZ {} has a 0 bytes LIP data.".format(
filename_fuz))
try:
with open(filename_xwm, "wb") as xwm_file:
xwm_file.write(fuz_file.read())
has_xwm = True
except:
util.LogInfo(
"ERROR: failed to create intermediate WMV <{}>.".
format(filename_xwm))
return False
util.LogDebug("INFO: XWM created on disk from FUZ {}.".format(
filename_fuz))
except:
util.LogInfo(
"ERROR: failed to open FUZ <{}>.".format(filename_lip))
return False
# Load LIP from disk in the rare case the LIP in FUZ is empty
if lip_size == 0 and has_lip:
try:
with open(filename_lip, "rb") as lip_file:
lip_data = lip_file.read()
lip_size = len(lip_data)
except:
util.LogInfo(
"ERROR: failed to open LIP <{}>.".format(filename_lip))
return False
# Convert the XWM to WAV
if has_xwm:
XWM2WAV(filename_xwm, filename_wav)
# Normalizes the WAV format
(err, channel_count,
sound_duration_ms) = WAV2PCM16WAV(filename_xwm, filename_wav, is_nxopus)
if err:
return False
# Converte the WAV to NXOPUS or MCADPCM
if is_nxopus:
WAV2NXOPUS(filename_wav, filename_nxopus)
else:
WAV2DSP(filename_wav, filename_dsp0, filename_dsp1, channel_count)
# write a FUZ container
if lip_size > 0 or is_nxopus:
lip_padding = lip_size % 4
if lip_padding != 0: lip_padding = 4 - lip_padding
voice_offset = 0x10 + lip_size + lip_padding
# write the FUZ header
#
# on PC a FUZ header has 0x0A bytes. NX adds an additional uint32_t to header end
# it represents the offset where the SOUND content starts as NX pads LIP content
#
fuz_nx_payload = BytesIO()
header_fuz = b'\x46\x55\x5A\x45\x01\x00\x00\x00'
fuz_nx_payload.write(header_fuz)
fuz_nx_payload.write(
lip_size.to_bytes(4, byteorder='little', signed=False))
fuz_nx_payload.write(
voice_offset.to_bytes(4, byteorder='little', signed=False))
# write the LIP content
if lip_size > 0: fuz_nx_payload.write(lip_data)
fuz_nx_payload.write(b'\x00' * lip_padding)
# write the SOUND content
if is_nxopus:
err = NXOPUS2FUZ(filename_nxopus, channel_count, sound_duration_ms,
fuz_nx_payload)
else:
err = DSP2MCADPCM(filename_dsp0, filename_dsp1, channel_count,
fuz_nx_payload)
if err:
return False
# pad the FUZ content to the closest uint32
fuz_padding = fuz_nx_payload.getbuffer().nbytes % 4
if fuz_padding != 0: fuz_padding = 4 - fuz_padding
fuz_nx_payload.write(b'\x00' * fuz_padding)
try:
with open(filename_fuz, "wb") as fuz_nx_file:
fuz_nx_file.write(fuz_nx_payload.getbuffer())
except:
util.LogInfo(
"ERROR: failed to create final FUZ <{}>.".format(filename_fuz))
return False
# write a MCADPCM container
else:
try:
with open(filename_mcadpcm, "wb") as mcadpcm_file:
DSP2MCADPCM(filename_dsp0, filename_dsp1, channel_count,
mcadpcm_file)
except:
util.LogInfo("ERROR: failed to create final MCADPCM <{}>.".format(
filename_mcadpcm))
return False
# clean up temporary files
util.RemoveFile(filename_wav)
if has_xwm:
util.RemoveFile(filename_xwm)
if has_lip:
util.RemoveFile(filename_lip)
if is_nxopus:
util.RemoveFile(filename_nxopus)
else:
util.RemoveFile(filename_dsp0)
if channel_count > 1:
util.RemoveFile(filename_dsp1)
if has_fuz and lip_size == 0:
util.RemoveFile(filename_fuz)
return True
def WriteIniFile(file, buffer):
newSkyrimIniFile = target + "\\" + file
util.LogInfo("Writing " + file)
with open(newSkyrimIniFile, "w+") as f:
f.write(buffer)
def LoadOrder(origin, target, loadOrderName):
targetData = target + r"\Data"
util.LogDebug("This is the origin: " + origin)
util.LogDebug("This is the target: " + target)
util.LogDebug("This is the target Data: " + targetData)
util.LogDebug("This is the load order name: " + loadOrderName)
util.LogInfo("Packed Load Order " + loadOrderName)
loadOrderTxt = os.path.join(origin, loadOrderName + ".txt")
loadOrder = open(loadOrderTxt, 'r').read()
#util.LogDebug("LOAD ORDER <" + loadOrder + ">")
loadOrderList = loadOrder.splitlines()
loadOrderStart = int(loadOrderList[0])
loadOrderList = loadOrderList[1:]
loadOrderList = list(filter(None, loadOrderList))
util.LogDebug("ESP Start at " + str(loadOrderStart))
util.LogDebug("LOAD ORDER LIST <" + str(loadOrderList) + ">")
pristineFolder = target
pristineSkyrimIni = pristineFolder + r"\Skyrim.ini"
util.LogDebug("Attempt to open " + pristineSkyrimIni)
pristineSkyrim = open(pristineSkyrimIni, 'r').read()
#util.LogDebug("PRISTINE:\n" + pristineSkyrim + "END PRISTINE")
newSkyrimIni = pristineSkyrim
languageInis = {}
for file in os.listdir(pristineFolder):
if file.endswith(".ini") and file != "Skyrim.ini" and file != "desktop.ini":
util.LogDebug("Language ini <" + file + ">")
languageInis[file] = pristineFolder + "\\" + file
util.LogDebug("Language Inis:")
for languageIni in languageInis:
liFilename = languageInis[languageIni]
util.LogDebug(languageIni+ " " + liFilename)
def GetArchiveList(aln, buffer, flags=None):
util.LogDebug("Find existing " + aln)
alPattern = "(" + aln + r"=[^\n$]*)[\n$]*"
if flags != None:
alPattern = re.compile(alPattern, flags)
al = re.search(alPattern, buffer)
if al == None:
util.LogError("Cannot find " + aln + ", bailing")
util.LogError("BUFFER" + buffer + "ENDBUFFER")
sys.exit(1)
retVal = al.group(1)
util.LogDebug("<" + retVal + ">")
return retVal
sResourceArchiveList = GetArchiveList("sResourceArchiveList", newSkyrimIni)
sResourceArchiveList2 = GetArchiveList("sResourceArchiveList2", newSkyrimIni)
sArchiveToLoadInMemoryList = GetArchiveList("sArchiveToLoadInMemoryList", newSkyrimIni)
def GetTestFile(tfn):
tfn = str(tfn)
util.LogDebug("Find test file " + tfn)
tfPattern = "([;]*sTestFile" + tfn + r"=[^\n]*)\n"
tf = re.search(tfPattern, newSkyrimIni)
if tf == None:
util.LogError("Cannot find " + tfn + ", bailing")
sys.exit(1)
retVal = tf.group(1)
util.LogDebug("<" + retVal + ">")
return retVal
CurrentTestFileIDX = loadOrderStart
sTestFiles = {}
for i in range(CurrentTestFileIDX, 11):
sTestFiles["sTestFile" + str(i)] = GetTestFile(i)
def CopyFile(file, filename):
newFileName = targetData + "\\" + file
util.LogDebug(filename + "->" + newFileName)
shutil.copy2(filename, newFileName)
def InsertTestFile(name, filename):
nonlocal CurrentTestFileIDX, newSkyrimIni
currentTestFile = "sTestFile" + str(CurrentTestFileIDX)
CurrentTestFileIDX = CurrentTestFileIDX + 1
newTestFile = currentTestFile + "=" + name
newSkyrimIni = newSkyrimIni.replace(sTestFiles[currentTestFile], newTestFile)
util.LogDebug(newTestFile)
CopyFile(file, filename)
#util.LogDebug("TESTFILES<" + str(sTestFiles) + ">")
resourceArchiveList2Additions = ''
newResourceArchiveList2 = sResourceArchiveList2
def InsertLanguageBSA(name, filename):
nonlocal resourceArchiveList2Additions, newResourceArchiveList2
resourceArchiveList2Additions += ", " + name
newResourceArchiveList2 += ", " + name
util.LogDebug(newResourceArchiveList2)
newResourceArchiveList = sResourceArchiveList
def InsertMainBSA(name, filename):
nonlocal newResourceArchiveList
newResourceArchiveList += ", " + name
util.LogDebug(newResourceArchiveList)
newArchiveToLoadInMemoryList = sArchiveToLoadInMemoryList
def InsertInMemoryBSA(name, filename):
nonlocal newResourceArchiveList, newArchiveToLoadInMemoryList
newResourceArchiveList += ", " + name
newArchiveToLoadInMemoryList += ", " + name
util.LogDebug(newResourceArchiveList)
util.LogDebug(newArchiveToLoadInMemoryList)
if loadOrderStart <= 4:
util.LogDebug("Hyrule.esp will not be loaded, so removing Skyrim - Hyrule.bsa from newResourceArchiveList")
newResourceArchiveList = newResourceArchiveList.replace(", Skyrim - Hyrule.bsa", "")
util.LogDebug("newResourceArchiveList is now:\n" + newResourceArchiveList)
iniPattern = r"^[; ]*([^=]*)=([^$]*)$"
def InsertIni(filename):
nonlocal newSkyrimIni
buffer = open(filename, 'r').read()
lines = buffer.splitlines()
pendingAdditions = []
currentHeader = ''
def EndHeader():
nonlocal newSkyrimIni
if len(pendingAdditions) > 0 and currentHeader != '':
currentHeaderSearch = currentHeader + "\n"
util.LogDebug("Search for <" + currentHeaderSearch + ">")
newHeader = currentHeaderSearch
for newKeyValue in pendingAdditions:
newHeader += newKeyValue
newSkyrimIni = newSkyrimIni.replace(currentHeaderSearch, newHeader)
for line in lines:
util.LogDebug("INI <" + line + ">")
if line.startswith("["):
EndHeader()
currentHeader = line
pendingAdditions = []
else:
newLine = line + "\n"
myKeySearch = re.search(iniPattern, line)
if myKeySearch == None:
util.LogDebug("INI line no key/pair found")
else:
myKey = myKeySearch.group(1)
myValue = myKeySearch.group(2)
util.LogDebug("MyKey <" + myKey + ">")
util.LogDebug("MyValue <" + myValue + ">")
localIniPattern = re.compile(r"^[; ]*" + myKey + r"=([^\n\r ]*)[ \n\r]", re.MULTILINE)
existingKeyValue = re.search(localIniPattern, newSkyrimIni)
if existingKeyValue != None:
wholePattern = existingKeyValue.group(0)
value = existingKeyValue.group(1)
util.LogDebug("***" + wholePattern + "->" + newLine + "***")
newSkyrimIni = newSkyrimIni.replace(wholePattern, newLine)
else:
pendingAdditions.append(newLine)
EndHeader()
for plugin in loadOrderList:
util.LogDebug("Found " + plugin)
pluginFolder = os.path.join(origin, plugin)
if plugin.startswith("#"):
util.LogDebug("Passing on " + plugin)
else:
unpack_mod.UnpackMod(pluginFolder, targetData)
for file in os.listdir(targetData):
if file.endswith(".ini"):
filename = os.path.join(targetData, file)
os.remove(filename)
for file in os.listdir(pluginFolder):
util.LogDebug(" Found " + file)
filename = os.path.join(pluginFolder, file)
if file.endswith(".esm"):
InsertTestFile(file, filename)
elif file.endswith(".ini"):
InsertIni(filename)
elif file.endswith(".bsa"):
pass # we unpacked already
else:
if os.path.isdir(filename):
shutil.copytree(filename, os.path.join(targetData, file))
else:
shutil.copy2(filename, os.path.join(targetData, file))
bsaList = pack_mod.PackMod(loadOrderName, targetData)
for filename in bsaList:
file = os.path.basename(filename)
if file.endswith("Textures.bsa"):
InsertLanguageBSA(file, filename)
elif file.endswith("Voices.bsa"):
InsertLanguageBSA(file, filename)
elif file.endswith("Animations.bsa"):
InsertInMemoryBSA(file, filename)
elif file.endswith(".bsa"):
InsertMainBSA(file, filename)
def WriteIniFile(file, buffer):
newSkyrimIniFile = target + "\\" + file
util.LogInfo("Writing " + file)
with open(newSkyrimIniFile, "w+") as f:
f.write(buffer)
newSkyrimIni = newSkyrimIni.replace(sResourceArchiveList, newResourceArchiveList)
newSkyrimIni = newSkyrimIni.replace(sResourceArchiveList2, newResourceArchiveList2)
newSkyrimIni = newSkyrimIni.replace(sArchiveToLoadInMemoryList, newArchiveToLoadInMemoryList)
WriteIniFile("Skyrim.ini", newSkyrimIni)
for languageIni in languageInis:
liFilename = languageInis[languageIni]
languageBuffer = open(liFilename, 'r').read()
util.LogDebug("opening " + liFilename)
liResourceArchiveList2 = GetArchiveList("sResourceArchiveList2", languageBuffer, re.MULTILINE)
newLiResourceArchiveList2 = liResourceArchiveList2 + resourceArchiveList2Additions
languageBuffer = languageBuffer.replace(liResourceArchiveList2, newLiResourceArchiveList2)
WriteIniFile(languageIni, languageBuffer)
# Convert Audio
if ok:
ok = ConvertAudio(filename_wav, filepath_without_extension, is_nxopus,
has_lip)
return ok
def ConvertSound(target, filepath_without_extension):
return ConvertSound_Internal(filepath_without_extension)
def ConvertSoundAsync(target, filename, logname, ret):
util.InitialiseMPLog(logname)
retVal = ConvertSound(target, filename)
ret["retVal"] = retVal
if __name__ == '__main__':
import sys
filepath = sys.argv[1]
util.SetScriptPath(sys.argv[2])
util.LogInfo("Skyrim-NX-Toolkit {} - convert_sound_zappa".format(
util.GetToolkitVersion()))
retValue = ConvertSound_Internal(filepath)
tempLog = util.GetTempLog()
for msg in tempLog:
print(msg)
if not retValue:
sys.exit(1)
sys.exit(0)
def PackMod(mod_name, target):
script_path = util.GetScriptPath()
utilities_path = util.GetUtilitiesPath()
bsarch = os.path.join(utilities_path, "bsarch.exe")
util.LogDebug("This is the target: " + target)
util.LogDebug("This is the mod name " + mod_name)
util.LogInfo("Pack Mod")
has_archive = util.HasArchive()
util.LogDebug("HasArchive is {}".format(has_archive))
data_list = os.listdir(target)
util.LogDebug(str(data_list))
BSARules = bsa_rules.GetBSARules()
BSAs = {}
SafePlugins = [
"skyrim.esm", "dawnguard.esm", "hearthfires.esm", "dragonborn.esm"
]
PluginPaths = {}
for plugin in SafePlugins:
PluginPaths[plugin] = plugin
# Arbitrary Limit to search for merged plugin list
LineLimit = 10
ChunkSize = 1024
ChunksLimit = 10
childPattern = re.compile(r"Merged Plugin:([^\0]*)[\0]", re.MULTILINE)
espPattern = re.compile(r"(.+?\.[^.]*$|$)", re.MULTILINE)
def ReadPlugin(filename):
util.LogDebug("Reading plugin{}".format(filename))
pluginList = []
with open(filename, "rb") as plugin:
lineNumber = 0
chunkNumber = 0
foundMerge = False
buffer = ''
while True:
chunk = plugin.read(ChunkSize)
chunkNumber += 1
buffer = buffer + "".join(map(chr, chunk))
if "Merged Plugin:" in buffer:
foundMerge = True
if foundMerge:
childPlugins = re.search(childPattern, buffer)
if childPlugins != None:
wholePattern = childPlugins.group(0)
value = childPlugins.group(1)
util.LogDebug("Found Plugins Block <{}>".format(value))
while True:
espTest = re.findall(espPattern, value)
if espTest != None:
for espCandidate in espTest:
espCandidate = espCandidate.strip(
) #''.join(espCandidate.split())
if espCandidate != '':
util.LogDebug(
"Found <{}>".format(espCandidate))
pluginList.append(espCandidate.lower())
#util.LogInfo("Found <{}>".format(str(espTest)))
break
break
if chunkNumber >= ChunksLimit:
break
if not foundMerge:
lineNumber += 1
if lineNumber >= LineLimit:
break
return pluginList
mod_pathname = mod_name + ".esp"
for root, subdirs, files in os.walk(target):
for file in files:
if file.endswith(".esp") or file.endswith(".esm"):
filename = os.path.join(root, file)
util.LogDebug("Found a plugin at {}, <{}>".format(
filename, file))
# look after yourself
PluginPaths[file.lower()] = file.lower()
childrenOfPlugin = ReadPlugin(filename)
if len(childrenOfPlugin) > 0:
util.LogInfo(
"Detected that {} is merged from:".format(file))
for child in childrenOfPlugin:
util.LogInfo(" - {}".format(child))
# look after your children
PluginPaths[child.lower()] = file.lower()
mod_pathname = file
# only interested in files in the root folder
break
util.LogDebug("PluginPaths is <{}>".format(str(PluginPaths)))
def DefineBSA(bsa_name):
nonlocal BSAs
temp = os.path.join(target, "Temp")
if bsa_name != '':
temp = os.path.join(target, "Temp - " + bsa_name)
temp_data = os.path.join(temp, "Data")
util.RemoveTree(temp)
os.makedirs(temp_data, exist_ok=True)
BSAs[bsa_name] = temp_data
numFoldersToMove = 0
numFilesToMove = 0
numFoldersMoved = 0
numFilesMoved = 0
def ApplyRuleToFolder(rule, folder):
nonlocal numFoldersMoved
bsa_name = rule["BSA"]
if bsa_name not in BSAs:
DefineBSA(bsa_name)
move_to_folder = BSAs[bsa_name]
child = os.path.basename(folder)
parent = os.path.dirname(folder)
if parent != target:
util.LogDebug("parent is not target")
rel_parent = os.path.relpath(parent, target)
util.LogDebug("rel_parent is {}".format(rel_parent))
move_to_folder = os.path.join(move_to_folder, rel_parent, child)
util.LogDebug("move_to_folder is {}".format(move_to_folder))
shutil.move(folder, move_to_folder)
util.LogDebug("moving {} to {}".format(folder, move_to_folder))
numFoldersMoved += 1
sys.stdout.write("Moved Files {}/{} Folders {}/{} \r".format(
numFilesMoved, numFilesToMove, numFoldersMoved, numFoldersToMove))
sys.stdout.flush()
def ApplyRuleToFile(rule, file_path):
nonlocal numFilesMoved
bsa_name = rule["BSA"]
if bsa_name not in BSAs:
DefineBSA(bsa_name)
relative_path = os.path.relpath(file_path, target)
target_path = os.path.join(BSAs[bsa_name], relative_path)
target_folder = os.path.dirname(target_path)
os.makedirs(target_folder, exist_ok=True)
util.LogDebug("moving {} to {}".format(file_path, target_path))
shutil.move(file_path, target_path)
numFilesMoved += 1
sys.stdout.write("Moved Files {}/{} Folders {}/{} \r".format(
numFilesMoved, numFilesToMove, numFoldersMoved, numFoldersToMove))
sys.stdout.flush()
ignoreMovedFolders = []
for root, subdirs, files in os.walk(target):
relative_folder = os.path.relpath(root, target).lower()
if root != target:
# Check if this folder is ignored already
ignoreThisFolder = False
for ignored in ignoreMovedFolders:
if relative_folder.startswith(ignored):
ignoreThisFolder = True
if not ignoreThisFolder:
# Now check if there is an unqualified path rule for this folder (like "scripts" are only placed in Misc)
folderCount = 0
lastRuleMatch = None
for rule in BSARules:
if "Folder" in rule and relative_folder.startswith(
rule["Folder"]):
folderCount += 1
lastRuleMatch = rule
if folderCount == 1:
numFoldersToMove += 1
ignoreMovedFolders.append(relative_folder)
else:
for filename in files:
util.LogDebug(os.path.join(relative_folder, filename))
numFilesToMove += 1
util.LogInfo("Files ({}) / Folders ({}) to move".format(
numFilesToMove, numFoldersToMove))
RemoveFolders = []
for root, subdirs, files in os.walk(target):
relative_folder = os.path.relpath(root, target).lower()
#util.LogDebug("Walking relative folder " + relative_folder)
if root == target:
for child in subdirs:
util.LogDebug("Children {}".format(child))
RemoveFolders.append(os.path.join(target, child))
else:
# First check if there is an unqualified path rule for this folder (like "scripts" are only placed in Misc)
folderCount = 0
lastRuleMatch = None
for rule in BSARules:
if "Folder" in rule and relative_folder.startswith(
rule["Folder"]):
folderCount += 1
lastRuleMatch = rule
if folderCount == 1:
util.LogDebug("ApplyRuleToFolder({}) -> {}".format(
lastRuleMatch["BSA"], relative_folder))
ApplyRuleToFolder(lastRuleMatch, root)
else:
#util.LogDebug("No folder match, check files")
for filename in files:
filename = filename.lower()
file_path = os.path.join(root, filename)
relative_path = os.path.relpath(file_path, target)
should_apply = False
for rule in BSARules:
should_apply = True
if should_apply and "Folder" in rule:
#util.LogDebug("checking folder rule {} vs relative_folder {}".format(rule["Folder"], relative_folder))
should_apply = should_apply and relative_folder.startswith(
rule["Folder"])
if should_apply and "Extension" in rule:
#util.LogDebug("checking extension rule {} vs filename {}".format(rule["Extension"], filename))
should_apply = should_apply and filename.endswith(
rule["Extension"])
if should_apply:
util.LogDebug("Applying BSA {} for {}".format(
rule["BSA"], file_path))
ApplyRuleToFile(rule, file_path)
break
if not should_apply:
util.LogWarn("Could not apply rule for <{}>".format(
relative_path))
for rule in BSARules:
should_apply = True
if should_apply and "Folder" in rule:
util.LogDebug(
"checking folder rule {} vs relative_folder {}"
.format(rule["Folder"], relative_folder))
should_apply = should_apply and relative_folder.startswith(
rule["Folder"])
if should_apply and "Extension" in rule:
util.LogDebug(
"checking extension rule {} vs filename {}"
.format(rule["Extension"], filename))
should_apply = should_apply and filename.endswith(
rule["Extension"])
sys.stdout.write("\n")
util.LogInfo("Cleanup old folders")
for folder in RemoveFolders:
util.LogDebug("Cleanup {}".format(folder))
util.RemoveTree(folder)
def CleanPluginSpecificPaths(cleanup_directory):
MoveFromTo = []
DeleteUnmatched = []
for root, subdirs, files in os.walk(cleanup_directory):
directory = root.lower()
dir_name = os.path.basename(directory)
if (dir_name.endswith("esm") or
dir_name.endswith("esp")) and dir_name not in SafePlugins:
if dir_name in PluginPaths:
target_pathname = PluginPaths[dir_name.lower()]
new_path = os.path.join(os.path.dirname(directory),
target_pathname)
if not os.path.isdir(new_path):
util.LogDebug(
"Rename plugin directory {} to {}".format(
directory, new_path))
os.rename(directory, new_path)
else:
util.LogDebug(
"Move plugin files from directory {} to {}".format(
directory, new_path))
MoveFromTo.append((directory, new_path))
else:
util.LogWarn("Marking <{}> for deletion.".format(dir_name))
DeleteUnmatched.append(directory)
for moveFromTo in MoveFromTo:
(move_from, move_to) = moveFromTo
util.LogDebug("Need to move from {} to {}".format(
move_from, move_to))
for root, subdirs, files in os.walk(move_from):
for file in files:
file_path = os.path.join(root, file)
relative_path = os.path.relpath(root, move_from)
new_path = os.path.join(move_to, relative_path, file)
util.LogDebug("Moving file from {} to {}".format(
file_path, new_path))
new_directory = os.path.dirname(new_path)
os.makedirs(new_directory, exist_ok=True)
shutil.move(file_path, new_path)
for deleteDirectory in DeleteUnmatched:
util.LogDebug("Deleting <{}>".format(deleteDirectory))
util.RemoveTree(deleteDirectory)
util.LogDebug("Deleted <{}>".format(deleteDirectory))
util.LogDebug("Build BSAs")
bsaList = []
for bsa_name in BSAs:
temp_data = BSAs[bsa_name]
temp = os.path.dirname(temp_data)
bsa_file_suffix = ""
if bsa_name != "":
bsa_file_suffix = " - " + bsa_name
CleanPluginSpecificPaths(temp_data)
bsa_filename = mod_name + bsa_file_suffix + ".bsa"
target_bsa = os.path.join(target, bsa_filename)
useArchive = has_archive
if useArchive:
bsa_list = archive_bsa.ArchiveBSA(temp, bsa_filename)
for bsa_info in bsa_list:
bsa_filename = bsa_info["FileName"]
bsa_filepath = bsa_info["Folder"]
bsa_fullpath = os.path.join(bsa_filepath, bsa_filename)
newTargetBSA = os.path.join(target, bsa_filename)
shutil.move(bsa_fullpath, newTargetBSA)
bsaList.append(newTargetBSA)
else:
bsa_list = bsarch_bsa.BsarchBSA(temp, target_bsa)
for bsa_info in bsa_list:
bsa_filename = bsa_info["FileName"]
bsa_filepath = bsa_info["Folder"]
bsa_fullpath = os.path.join(bsa_filepath, bsa_filename)
newTargetBSA = os.path.join(target, bsa_filename)
shutil.move(bsa_fullpath, newTargetBSA)
bsaList.append(newTargetBSA)
util.RemoveTree(temp)
util.LogDebug("PackMod Done")
return bsaList
def ArchiveBSA(target_folder, bsa_filename):
script_path = util.GetScriptPath()
utilities_path = util.GetUtilitiesPath()
archive_original = os.path.join(utilities_path, "Archive.exe")
util.LogDebug("Copy Archive.exe to target folder")
archive = os.path.join(target_folder, "Archive.exe")
shutil.copy2(archive_original, archive)
log_basename = "log.txt"
log_filename = os.path.join(target_folder, log_basename)
config_basename = "bsa_config.txt"
config_filename = os.path.join(target_folder, config_basename)
allFilesList = []
checks = {}
util.LogInfo("Build File List")
totalFileSizeTally = 0
target_data = os.path.join(target_folder, "Data")
util.LogDebug("Walking the target directory " + target_data)
for root, subdirs, files in os.walk(target_data):
util.LogDebug('--\nroot = ' + root)
if root == target_data:
util.LogDebug("subdirs: " + str(subdirs))
lower_case_data_list = [x.lower() for x in subdirs]
util.LogDebug("lcds: " + str(lower_case_data_list))
if "meshes" in lower_case_data_list:
util.LogDebug("found meshes")
checks["Retain Strings During Startup"] = True
checks["Meshes"] = True
if "textures" in lower_case_data_list:
util.LogDebug("found texttures")
checks["Textures"] = True
if "interface" in lower_case_data_list:
util.LogDebug("found interface")
checks["Menus"] = True
if "music" in lower_case_data_list:
util.LogDebug("found music")
checks["Retain File Names"] = True
checks["Sounds"] = True
if "sound" in lower_case_data_list:
util.LogDebug("found sound")
sound_list = os.listdir(os.path.join(target_data, "sound"))
sound_list_lower = [x.lower() for x in sound_list]
if "fx" in sound_list_lower:
util.LogDebug("found sound//fx")
checks["Retain File Names"] = True
checks["Sounds"] = True
if "voice" in sound_list_lower:
util.LogDebug("found sound//voice")
checks["Voices"] = True
if "shadersfx" in lower_case_data_list:
util.LogDebug("found shaders")
checks["Shaders"] = True
if "seq" in lower_case_data_list:
util.LogDebug("found seq")
checks["Retain File Names"] = True
checks["Misc"] = True
if "grass" in lower_case_data_list:
util.LogDebug("found grass")
checks["Retain File Names"] = True
checks["Misc"] = True
if "scripts" in lower_case_data_list:
util.LogDebug("found scripts")
checks["Retain File Names"] = True
checks["Misc"] = True
else:
for filename in files:
if filename != "desktop.ini":
file_path = os.path.join(root, filename)
relative_path = file_path.replace(target_folder, '')
util.LogDebug('\t- file %s (relative path: %s)' %
(filename, relative_path))
path_no_data = relative_path[6:]
file_size = os.path.getsize(file_path)
totalFileSizeTally += file_size
util.LogDebug("totalFileSizeTally is now: " +
str(totalFileSizeTally))
allFilesList.append({
'FileName': filename,
'FilePath': file_path,
'RelativePath': relative_path,
'PathNoData': path_no_data,
'FileSize': file_size
})
SizeLimitBSA = bsa_rules.BSASizeLimit
currentFileIndex = None
currentFileSizeTally = 0
buffer = ''
bsaFileWritten = []
bsa_original_filename = bsa_filename
def WrtiteBSA():
nonlocal currentFileIndex, buffer, bsa_filename
util.LogDebug("Writing BSA with filelist:<" + buffer + ">")
filelist_basename = "bsa_filelist.txt"
if currentFileIndex != None:
filelist_basename = "bsa_filelist" + str(currentFileIndex) + ".txt"
bsa_filename = bsa_original_filename[:-4] + str(
currentFileIndex) + ".bsa"
currentFileIndex += 1
filelist_filename = os.path.join(target_folder, filelist_basename)
with open(filelist_filename, 'w') as filelist_file:
filelist_file.write(buffer)
buffer = ''
util.LogInfo("Build Config")
checksOrder = [
"Meshes", "Textures", "Menus", "Sounds", "Voices", "Shaders",
"Trees", "Fonts", "Misc", "Compress Archive",
"Retain Directory Names", "Retain File Names",
"Retain File Name Offsets", "Retain Strings During Startup",
"XBox 360 Archive", "Embed File Names"
]
with open(config_filename, 'w') as config_file:
config_file.write("Log: " + log_basename + "\n")
config_file.write("New Archive\n")
for check in checksOrder:
if check in checks:
config_file.write("Check: " + check + "\n")
config_file.write("Set File Group Root: Data\\\n")
config_file.write("Add File Group: " + filelist_basename + "\n")
config_file.write("Save Archive: " + bsa_filename + "\n")
util.LogInfo("Run Archive.exe")
commandLine = ["Archive.exe", config_basename]
os.chdir(target_folder)
util.RunCommandLine(commandLine)
with open(log_filename, "r") as log_file:
for line in log_file:
util.LogDebug(line)
os.remove(log_filename)
os.remove(filelist_filename)
os.remove(config_filename)
bsaFileWritten.append({
"Folder": target_folder,
"FileName": bsa_filename
})
if totalFileSizeTally > SizeLimitBSA:
currentFileIndex = 0
totalWrittenTally = 0
for fileInfo in allFilesList:
file_size = fileInfo['FileSize']
newTally = currentFileSizeTally + file_size
totalWrittenTally = totalWrittenTally + file_size
util.LogDebug("Adding " + fileInfo['FileName'] +
" currentFileSizeTally is " + str(currentFileSizeTally) +
" file_size is " + str(file_size) +
" totalWrittenTally is " + str(totalWrittenTally))
buffer += fileInfo['PathNoData'] + "\n"
currentFileSizeTally += file_size
if (newTally >= SizeLimitBSA) or (totalWrittenTally >=
totalFileSizeTally):
WrtiteBSA()
currentFileSizeTally = 0
if buffer != '':
util.LogWarn("BUFFER NOT EMPTY!")
util.LogInfo("Clean Up")
util.RemoveFile(archive)
os.chdir(script_path)
return bsaFileWritten
def ConvertSound_Internal(filepath_without_extension):
""" Converts PC SSE sound files to be compatible with NX SSE supported codecs """
filename_wav = filepath_without_extension + ".wav"
filename_xwm = filepath_without_extension + ".xwm"
filename_lip = filepath_without_extension + ".lip"
filename_fuz = filepath_without_extension + ".fuz"
has_wav = os.path.exists(filename_wav)
has_xwm = os.path.exists(filename_xwm)
has_lip = os.path.exists(filename_lip)
has_fuz = os.path.exists(filename_fuz)
util.LogDebug(
"INFO: Convert Sound <{}> WAV:{} XWM:{} LIP:{} FUZ:{}".format(
filepath_without_extension, has_wav, has_xwm, has_lip, has_fuz))
# UNFUZ Audio
if has_fuz:
try:
with open(filename_fuz, "rb") as fuz_file:
fuz_file.seek(0x08)
lip_size = int.from_bytes(fuz_file.read(0x04),
byteorder='little',
signed=False)
lip_data = fuz_file.read(lip_size)
audio_data = fuz_file.read()
except:
util.LogInfo(
"ERROR: failed to open FUZ <{}>.".format(filename_lip))
return False
# determine AUDIO format
audio_format = audio_data[0x08:0x0C]
if audio_format == b'WAVE':
has_wav = True
filename_audio = filename_wav
elif audio_format == b'XWMA':
has_xwm = True
filename_audio = filename_xwm
else:
util.LogInfo("ERROR: unknown audio format {} on FUZ <{}>.".format(
audio_format, filename_fuz))
return False
# save LIP contents
if lip_size > 0:
try:
with open(filename_lip, "wb") as lip_file:
lip_file.write(lip_data)
has_lip = True
util.LogDebug(
"INFO: LIP created on disk from FUZ {}.".format(
filename_fuz))
except:
util.LogDebug(
"ERROR: failed to create intermediate LIP <{}>.".format(
filename_lip))
return False
# save AUDIO contents
try:
with open(filename_audio, "wb") as audio_file:
audio_file.write(audio_data)
util.LogDebug(
"INFO: AUDIO created on disk from FUZ {}.".format(
filename_fuz))
except:
util.LogDebug(
"ERROR: failed to create intermediate AUDIO <{}>.".format(
filename_audio))
return False
# get rid of the source PC FUZ file
util.RemoveFile(filename_fuz)
elif has_xwm:
filename_audio = filename_xwm
elif has_wav:
filename_audio = filename_wav
else:
util.LogDebug("PANIC: IT SHOULD NEVER REACH THIS BRANCH...")
return False
# Force anything VOICE to use OPUS codec
is_nxopus = "\\sound\\voice\\" in filepath_without_extension.lower()
# Normalize Audio
ok = NormalizeAudio(filename_audio, filepath_without_extension, is_nxopus)
# Convert Audio
if ok:
ok = ConvertAudio(filename_wav, filepath_without_extension, is_nxopus,
has_lip)
return ok
