def InsertInMemoryBSA(name, filename):
nonlocal newResourceArchiveList, newArchiveToLoadInMemoryList
newResourceArchiveList += ", " + name
newArchiveToLoadInMemoryList += ", " + name
util.LogDebug(newResourceArchiveList)
util.LogDebug(newArchiveToLoadInMemoryList)
CopyFile(file, filename)
def CheckHKX_Internal(filename):
util.LogDebug("CheckHKX : " + filename)
with open(filename, "rb") as file:
file.seek(0, 0)
fiveBytes = file.read(5)
isXML = True
for i in range(5):
if Magic_XML[i] != fiveBytes[i]:
isXML = False
break
if isXML:
return PC_XML
file.seek(16, 0)
bitsFlag = file.read(1)
file.seek(18, 0)
nxFlag = file.read(1)
util.LogDebug("BF {} NF {}".format(bitsFlag, nxFlag))
if bitsFlag == b'\x04':
if nxFlag == b'\x00':
return PC_32
elif bitsFlag == b'\x08':
if nxFlag == b'\x00':
return PC_64
elif nxFlag == b'\x01':
return NX_64
return Unknown
def MakeTest(origin, target):
script_path = util.GetScriptPath()
util.LogDebug("This is the origin: " + origin)
util.LogDebug("This is the target: " + target)
FoldersToCreate = []
FilesToCreate = []
util.CreateTarget(target)
util.LogDebug("Target Created")
for root, subdirs, files in os.walk(origin):
if root != origin:
FoldersToCreate.append(root)
util.LogDebug("Adding folder ({})".format(root))
for filename in files:
file_path = os.path.join(root, filename)
FilesToCreate.append(file_path)
util.LogDebug("Adding file ({})".format(file_path))
for folderToCreate in FoldersToCreate:
util.LogDebug("About to add folder ({})".format(folderToCreate))
newFolderName = folderToCreate.replace(origin, target)
util.LogDebug("New folder name is ({})".format(newFolderName))
os.mkdir(newFolderName)
for fileToCreate in FilesToCreate:
util.LogDebug("About to add file ({})".format(fileToCreate))
newFileName = fileToCreate.replace(origin, target)
util.LogDebug("New filename is ({})".format(newFileName))
with open(newFileName, "w") as myFile:
myFile.write("TESTFILE")
def GetTestFile(tfn):
tfn = str(tfn)
util.LogDebug("Find test file " + tfn)
tfPattern = "([;]*sTestFile" + tfn + r"=[^\n]*)\n"
tf = re.search(tfPattern, newSkyrimIni)
if tf == None:
util.LogError("Cannot find " + tfn + ", bailing")
sys.exit(1)
retVal = tf.group(1)
util.LogDebug("<" + retVal + ">")
return retVal
def GetArchiveList(aln, buffer, flags=None):
util.LogDebug("Find existing " + aln)
alPattern = "(" + aln + r"=[^\n$]*)[\n$]*"
if flags != None:
alPattern = re.compile(alPattern, flags)
al = re.search(alPattern, buffer)
if al == None:
util.LogError("Cannot find " + aln + ", bailing")
util.LogError("BUFFER" + buffer + "ENDBUFFER")
sys.exit(1)
retVal = al.group(1)
util.LogDebug("<" + retVal + ">")
return retVal
def InsertIni(filename):
nonlocal newSkyrimIni
buffer = open(filename, 'r').read()
lines = buffer.splitlines()
pendingAdditions = []
currentHeader = ''
def EndHeader():
nonlocal newSkyrimIni
if len(pendingAdditions) > 0 and currentHeader != '':
currentHeaderSearch = currentHeader + "\n"
util.LogDebug("Search for <" + currentHeaderSearch + ">")
newHeader = currentHeaderSearch
for newKeyValue in pendingAdditions:
newHeader += newKeyValue
newSkyrimIni = newSkyrimIni.replace(currentHeaderSearch,
newHeader)
for line in lines:
util.LogDebug("INI <" + line + ">")
if line.startswith("["):
EndHeader()
currentHeader = line
pendingAdditions = []
else:
newLine = line + "\n"
myKeySearch = re.search(iniPattern, line)
if myKeySearch == None:
util.LogDebug("INI line no key/pair found")
else:
myKey = myKeySearch.group(1)
myValue = myKeySearch.group(2)
util.LogDebug("MyKey <" + myKey + ">")
util.LogDebug("MyValue <" + myValue + ">")
localIniPattern = re.compile(
r"^[; ]*" + myKey + r"=([^\n\r ]*)[ \n\r]",
re.MULTILINE)
existingKeyValue = re.search(localIniPattern, newSkyrimIni)
if existingKeyValue != None:
wholePattern = existingKeyValue.group(0)
value = existingKeyValue.group(1)
util.LogDebug("***" + wholePattern + "->" + newLine +
"***")
newSkyrimIni = newSkyrimIni.replace(
wholePattern, newLine)
else:
pendingAdditions.append(newLine)
EndHeader()
def ConvertHKX64_Internal(filename):
util.LogDebug("ConvertHKX64 : " + filename)
with open(filename, "rb") as hkx64:
payload = hkx64.read()
header_location = payload.find(b'\x00' * 15 + b'\x80' + b'\x00' * 48)
is_skeleton_container = payload.find(b'hkaSkeleton') > 0
is_behavior_container = payload.find(b'hkbBehavior') > 0
# it should not happen here as convert_path already doing some checks
if is_behavior_container or is_skeleton_container:
util.LogInfo("Warning, cannot convert {} because <{}>".format(
filename, 'SSE hkx animation'))
return False
# swap some fields in the AMD64 HKX header structure to make it PS4 compatible
if header_location > 0:
is_project_data = payload.find(b'hkbProjectData')
offset = 72 if is_project_data > 0 else 76
with open(filename, "wb") as hkxPS4:
hkxPS4.write(payload[0:18])
hkxPS4.write(b'\x01')
hkxPS4.write(payload[19:header_location + 16])
hkxPS4.write(payload[header_location + 20:header_location +
offset])
hkxPS4.write(payload[header_location:header_location + 4])
hkxPS4.write(payload[header_location + offset:])
return True
def ConvertAudio(filename_input_audio, filepath_without_extension, is_nxopus,
has_lip):
""" converts the audio file to final sse nx format """
VGAudioCli = GetVGAudioCli()
if is_nxopus and has_lip:
filename_lip = filepath_without_extension + ".lip"
filename_temp_lip = filepath_without_extension + ".temp.lip"
util.RenameFile(filename_lip, filename_temp_lip)
if is_nxopus:
filename_temp = filepath_without_extension + ".temp.fuz"
filename_output = filepath_without_extension + ".fuz"
commandLine = [
VGAudioCli, "-c", "--opusheader", "Skyrim", "-i:0",
filename_input_audio, filename_temp
]
else:
filename_temp = filepath_without_extension + ".temp.mcadpcm"
filename_output = filepath_without_extension + ".mcadpcm"
commandLine = [VGAudioCli, "-c", filename_input_audio, filename_temp]
util.RunCommandLine(commandLine)
try:
if is_nxopus and has_lip:
util.RemoveFile(filename_temp_lip)
util.RemoveFile(filename_input_audio)
util.RenameFile(filename_temp, filename_output)
util.LogDebug("INFO: Converted AUDIO <{}>".format(filename_output))
return True
except:
return False
def NormalizeAudio(filename_input_audio, filepath_without_extension,
is_nxopus):
""" Normalizes the audio file to be a proper PCM16 with a correct sample rate for VGAudioCli """
FFMpeg = GetFFMpeg()
filename_temp = filepath_without_extension + ".temp.wav"
filename_output = filepath_without_extension + ".wav"
if is_nxopus:
commandLine = [
FFMpeg, "-hide_banner", "-y", "-i", filename_input_audio, "-ac",
"1", "-ar", "48000", filename_temp
]
else:
commandLine = [
FFMpeg, "-hide_banner", "-y", "-i", filename_input_audio, "-ar",
"44100", filename_temp
]
util.RunCommandLine(commandLine)
try:
util.RemoveFile(filename_input_audio)
util.RenameFile(filename_temp, filename_output)
util.LogDebug("INFO: Normalized AUDIO <{}>".format(filename_output))
return True
except:
return False
def WriteBSA():
nonlocal currentFileIndex, bsa_filename
util.LogDebug("Writing BSA")
if currentFileIndex != None:
bsa_filename = bsa_original_filename[:-4] + str(
currentFileIndex) + ".bsa"
currentFileIndex += 1
util.LogInfo("Run bsarch.exe")
flags_value = flags.GetValue()
flags_hexvalue = hex(flags_value)
compress = "-z" if flags.IsSet(Flag_Compressed) else ""
commandLine = [
bsarch, "pack", temp_data, bsa_filename, "-sse", compress,
"-af:" + flags_hexvalue
]
util.RunCommandLine(commandLine)
bsaFileWritten.append({
"Folder": target_folder,
"FileName": bsa_filename
})
util.RemoveTree(temp_data)
def WrtiteBSA():
nonlocal currentFileIndex, buffer, bsa_filename
util.LogDebug("Writing BSA with filelist:<" + buffer + ">")
filelist_basename = "bsa_filelist.txt"
if currentFileIndex != None:
filelist_basename = "bsa_filelist" + str(currentFileIndex) + ".txt"
bsa_filename = bsa_original_filename[:-4] + str(
currentFileIndex) + ".bsa"
currentFileIndex += 1
filelist_filename = os.path.join(target_folder, filelist_basename)
with open(filelist_filename, 'w') as filelist_file:
filelist_file.write(buffer)
buffer = ''
util.LogInfo("Build Config")
checksOrder = [
"Meshes", "Textures", "Menus", "Sounds", "Voices", "Shaders",
"Trees", "Fonts", "Misc", "Compress Archive",
"Retain Directory Names", "Retain File Names",
"Retain File Name Offsets", "Retain Strings During Startup",
"XBox 360 Archive", "Embed File Names"
]
with open(config_filename, 'w') as config_file:
config_file.write("Log: " + log_basename + "\n")
config_file.write("New Archive\n")
for check in checksOrder:
if check in checks:
config_file.write("Check: " + check + "\n")
config_file.write("Set File Group Root: Data\\\n")
config_file.write("Add File Group: " + filelist_basename + "\n")
config_file.write("Save Archive: " + bsa_filename + "\n")
util.LogInfo("Run Archive.exe")
commandLine = ["Archive.exe", config_basename]
os.chdir(target_folder)
util.RunCommandLine(commandLine)
with open(log_filename, "r") as log_file:
for line in log_file:
util.LogDebug(line)
os.remove(log_filename)
os.remove(filelist_filename)
os.remove(config_filename)
bsaFileWritten.append({
"Folder": target_folder,
"FileName": bsa_filename
})
def InsertTestFile(name, filename): nonlocal CurrentTestFileIDX, newSkyrimIni currentTestFile = "sTestFile" + str(CurrentTestFileIDX) CurrentTestFileIDX = CurrentTestFileIDX + 1 newTestFile = currentTestFile + "=" + name newSkyrimIni = newSkyrimIni.replace(sTestFiles[currentTestFile], newTestFile) util.LogDebug(newTestFile) CopyFile(file, filename)
def EndHeader():
nonlocal newSkyrimIni
if len(pendingAdditions) > 0 and currentHeader != '':
currentHeaderSearch = currentHeader + "\n"
util.LogDebug("Search for <" + currentHeaderSearch + ">")
newHeader = currentHeaderSearch
for newKeyValue in pendingAdditions:
newHeader += newKeyValue
newSkyrimIni = newSkyrimIni.replace(currentHeaderSearch, newHeader)
def getIntPattern(name):
namePattern = name + r": ([\d]+)"
nameReturn = re.search(namePattern, dds_buffer)
if nameReturn != None:
nameReturn = nameReturn.group(1)
else:
nameReturn = -1
util.LogDebug(name + " is :" + str(nameReturn))
return int(nameReturn)
def ConvertHKX_Internal(filename):
util.LogDebug("ConvertHKX : " + filename)
havocBPP = util.GetHavokBPP()
commandLine = [
havocBPP, "--stripMeta", "--platformPS4", filename, filename
]
util.RunCommandLine(commandLine)
return True
def ApplyRuleToFolder(rule, folder):
nonlocal numFoldersMoved
bsa_name = rule["BSA"]
if bsa_name not in BSAs:
DefineBSA(bsa_name)
move_to_folder = BSAs[bsa_name]
child = os.path.basename(folder)
parent = os.path.dirname(folder)
if parent != target:
util.LogDebug("parent is not target")
rel_parent = os.path.relpath(parent, target)
util.LogDebug("rel_parent is {}".format(rel_parent))
move_to_folder = os.path.join(move_to_folder, rel_parent, child)
util.LogDebug("move_to_folder is {}".format(move_to_folder))
shutil.move(folder, move_to_folder)
util.LogDebug("moving {} to {}".format(folder, move_to_folder))
numFoldersMoved += 1
sys.stdout.write("Moved Files {}/{} Folders {}/{} \r".format(
numFilesMoved, numFilesToMove, numFoldersMoved, numFoldersToMove))
sys.stdout.flush()
def ReadPlugin(filename):
util.LogDebug("Reading plugin{}".format(filename))
pluginList = []
with open(filename, "rb") as plugin:
lineNumber = 0
chunkNumber = 0
foundMerge = False
buffer = ''
while True:
chunk = plugin.read(ChunkSize)
chunkNumber += 1
buffer = buffer + "".join(map(chr, chunk))
if "Merged Plugin:" in buffer:
foundMerge = True
if foundMerge:
childPlugins = re.search(childPattern, buffer)
if childPlugins != None:
wholePattern = childPlugins.group(0)
value = childPlugins.group(1)
util.LogDebug("Found Plugins Block <{}>".format(value))
while True:
espTest = re.findall(espPattern, value)
if espTest != None:
for espCandidate in espTest:
espCandidate = espCandidate.strip(
) #''.join(espCandidate.split())
if espCandidate != '':
util.LogDebug(
"Found <{}>".format(espCandidate))
pluginList.append(espCandidate.lower())
#util.LogInfo("Found <{}>".format(str(espTest)))
break
break
if chunkNumber >= ChunksLimit:
break
if not foundMerge:
lineNumber += 1
if lineNumber >= LineLimit:
break
return pluginList
def RepackMod(origin, target):
mod_name = os.path.basename(origin)
'''
util.LogDebug("This is the origin: " + origin)
util.LogDebug("This is the target: " + target)
util.LogDebug("This is the mod name " + mod_name)
'''
util.LogDebug("convert_mod.py 2.0")
util.LogInfo("Convert Mod, create empty folder at target")
util.CreateTarget(target)
unpack_mod.UnpackMod(origin, target)
pack_mod.PackMod(mod_name, target)
def Update(self):
self.m_UpdateCount += 1
#util.LogDebug("{} JM Update {}/{}".format(self.m_UpdateCount, len(self.m_RunningJobs), len(self.m_JobsBacklog)))
for job in list(self.m_RunningJobs):
if job.IsFinished():
self.m_RunningJobs.remove(job)
util.LogDebug("{} JM Removing Finished Job {}/{}".format(
self.m_UpdateCount, len(self.m_RunningJobs),
len(self.m_JobsBacklog)))
while len(self.m_RunningJobs) < self.m_MaxThreads and len(
self.m_JobsBacklog) > 0:
job = self.m_JobsBacklog.pop(0)
def JobWorker():
job.Run()
newThread = threading.Thread(target=JobWorker)
job.SetThread(newThread)
newThread.start()
self.m_RunningJobs.append(job)
util.LogDebug("{} JM Started Job From Backlog {}/{}".format(
self.m_UpdateCount, len(self.m_RunningJobs),
len(self.m_JobsBacklog)))
def ApplyRuleToFile(rule, file_path):
nonlocal numFilesMoved
bsa_name = rule["BSA"]
if bsa_name not in BSAs:
DefineBSA(bsa_name)
relative_path = os.path.relpath(file_path, target)
target_path = os.path.join(BSAs[bsa_name], relative_path)
target_folder = os.path.dirname(target_path)
os.makedirs(target_folder, exist_ok=True)
util.LogDebug("moving {} to {}".format(file_path, target_path))
shutil.move(file_path, target_path)
numFilesMoved += 1
sys.stdout.write("Moved Files {}/{} Folders {}/{} \r".format(
numFilesMoved, numFilesToMove, numFoldersMoved, numFoldersToMove))
sys.stdout.flush()
def ConvertTXT_Internal(filename):
buffer = None
with open(filename, "rb") as pcFile:
buffer = pcFile.read()
buffer = bytearray(buffer)
util.LogInfo("Buffer0<{}>".format(buffer[0]))
util.LogInfo("Buffer0<{}>".format(buffer[1]))
i = 1
while i < len(buffer):
buffer[i:i] = b'\x00\x00\x00'
i += 4
util.LogDebug("BUFFER<{}>".format(buffer))
with open(filename, "wb") as outFile:
outFile.write(b'\xff\xfe\x00\x00')
outFile.write(buffer)
return True
def ConvertMod(origin, target, oldrim):
mod_name = os.path.basename(origin)
'''
util.LogDebug("This is the origin: " + origin)
util.LogDebug("This is the target: " + target)
util.LogDebug("This is the mod name " + mod_name)
'''
util.LogDebug("convert_mod.py 2.0")
toolkit_path = util.GetToolKitPath()
util.LogInfo("Convert Mod, ToolkitPath is {}".format(toolkit_path))
util.LogInfo("Convert Mod, create empty folder at target")
util.CreateTarget(target)
unpack_mod.UnpackMod(origin, target)
if oldrim:
reconcile_hkx.ReconcileHKX(target, oldrim)
convert_path.ConvertPath(mod_name, target)
pack_mod.PackMod(mod_name, target)
def ConvertNIF_Internal(filename):
utilities_path = util.GetUtilitiesPath()
nswnifopt = os.path.join(utilities_path, "nswnifopt.exe")
util.LogDebug("ConvertNIF_Internal: " + " " + filename)
removeEditorMarker = "--remove-editor-marker" if toolkit_config.get_bool_setting(
"Meshes", "RemoveEditorMarker") else ""
prettySortBlocks = "--pretty-sort-blocks" if toolkit_config.get_bool_setting(
"Meshes", "PrettySortBlocks") else ""
trimTexturesPath = "--trim-textures-path" if toolkit_config.get_bool_setting(
"Meshes", "TrimTexturesPath") else ""
optimizeForSSE = "--optimize-for-sse" if toolkit_config.get_bool_setting(
"Meshes", "OptimizeForSSE") else ""
commandLine = [
nswnifopt, "-i", filename, "-o", filename, removeEditorMarker,
prettySortBlocks, trimTexturesPath, optimizeForSSE
]
util.RunCommandLine(commandLine)
return True
def CleanPluginSpecificPaths(cleanup_directory):
MoveFromTo = []
DeleteUnmatched = []
for root, subdirs, files in os.walk(cleanup_directory):
directory = root.lower()
dir_name = os.path.basename(directory)
if (dir_name.endswith("esm") or
dir_name.endswith("esp")) and dir_name not in SafePlugins:
if dir_name in PluginPaths:
target_pathname = PluginPaths[dir_name.lower()]
new_path = os.path.join(os.path.dirname(directory),
target_pathname)
if not os.path.isdir(new_path):
util.LogDebug(
"Rename plugin directory {} to {}".format(
directory, new_path))
os.rename(directory, new_path)
else:
util.LogDebug(
"Move plugin files from directory {} to {}".format(
directory, new_path))
MoveFromTo.append((directory, new_path))
else:
util.LogWarn("Marking <{}> for deletion.".format(dir_name))
DeleteUnmatched.append(directory)
for moveFromTo in MoveFromTo:
(move_from, move_to) = moveFromTo
util.LogDebug("Need to move from {} to {}".format(
move_from, move_to))
for root, subdirs, files in os.walk(move_from):
for file in files:
file_path = os.path.join(root, file)
relative_path = os.path.relpath(root, move_from)
new_path = os.path.join(move_to, relative_path, file)
util.LogDebug("Moving file from {} to {}".format(
file_path, new_path))
new_directory = os.path.dirname(new_path)
os.makedirs(new_directory, exist_ok=True)
shutil.move(file_path, new_path)
for deleteDirectory in DeleteUnmatched:
util.LogDebug("Deleting <{}>".format(deleteDirectory))
util.RemoveTree(deleteDirectory)
util.LogDebug("Deleted <{}>".format(deleteDirectory))
def ReconcileHKX(mod_path, oldrim_path):
script_path = util.GetScriptPath()
util.LogInfo("Reconcile HKX")
util.LogDebug("This is the mod_path: " + mod_path)
util.LogDebug("This is the oldrim_path " + oldrim_path)
CopyHKX = []
totalCount = 0
matchedCount = 0
for root, subdirs, files in os.walk(mod_path):
util.LogDebug("Walking folder " + root)
for filename in files:
if filename.lower().endswith(".hkx"):
file_path = os.path.join(root, filename)
relative_path = os.path.relpath(file_path, mod_path)
util.LogDebug("Relative path {} OR Path {}".format(
relative_path, oldrim_path))
oldrim_file_path = os.path.join(oldrim_path, relative_path)
totalCount += 1
util.LogDebug("Found {}, checking {}".format(
file_path, oldrim_file_path))
if os.path.exists(oldrim_file_path):
util.LogDebug(
"Found {} match in oldrim".format(oldrim_file_path))
matchedCount += 1
CopyHKX.append((file_path, oldrim_file_path))
util.LogInfo("Matched {}/{} hkx files in the mod".format(
matchedCount, totalCount))
for i in range(len(CopyHKX)):
(copy_to, copy_from) = CopyHKX[i]
util.LogDebug("Copying {}->{}".format(copy_from, copy_to))
shutil.copy2(copy_from, copy_to)
sys.stdout.write("Reconciled {}/{} \r".format(i + 1, len(CopyHKX)))
sys.stdout.flush()
sys.stdout.write("\n")
def readDDS(f, SRGB):
with open(f, "rb") as inf:
inb = inf.read()
if len(inb) < 0x80 or inb[:4] != b'DDS ':
util.LogDebug("")
util.LogDebug(f + " is not a valid DDS file!")
return 0, 0, 0, b'', 0, [], 0, []
width = struct.unpack("<I", inb[16:20])[0]
height = struct.unpack("<I", inb[12:16])[0]
fourcc = inb[84:88]
if fourcc == b'DX10':
util.LogDebug("")
util.LogDebug("DX10 DDS files are not supported.")
return 0, 0, 0, b'', 0, [], 0, []
pflags = struct.unpack("<I", inb[80:84])[0]
bpp = struct.unpack("<I", inb[88:92])[0] >> 3
channel0 = struct.unpack("<I", inb[92:96])[0]
channel1 = struct.unpack("<I", inb[96:100])[0]
channel2 = struct.unpack("<I", inb[100:104])[0]
channel3 = struct.unpack("<I", inb[104:108])[0]
caps = struct.unpack("<I", inb[108:112])[0]
if caps not in [0x1000, 0x401008]:
util.LogDebug("")
util.LogDebug("Invalid texture (caps[" + str(caps) +
"] not in [0x1000, 0x401008]).")
return 0, 0, 0, b'', 0, [], 0, []
abgr8_masks = {0xff: 0, 0xff00: 1, 0xff0000: 2, 0xff000000: 3, 0: 5}
bgr8_masks = {0xff: 0, 0xff00: 1, 0xff0000: 2, 0: 5}
a2rgb10_masks = {0x3ff00000: 0, 0xffc00: 1, 0x3ff: 2, 0xc0000000: 3, 0: 5}
bgr565_masks = {0x1f: 0, 0x7e0: 1, 0xf800: 2, 0: 5}
a1bgr5_masks = {0x1f: 0, 0x3e0: 1, 0x7c00: 2, 0x8000: 3, 0: 5}
abgr4_masks = {0xf: 0, 0xf0: 1, 0xf00: 2, 0xf000: 3, 0: 5}
l8_masks = {0xff: 0, 0: 5}
a8l8_masks = {0xff: 0, 0xff00: 1, 0: 5}
compressed = False
luminance = False
rgb = False
has_alpha = False
if pflags == 4:
compressed = True
elif pflags == 0x20000 or pflags == 2:
luminance = True
elif pflags == 0x20001:
luminance = True
has_alpha = True
elif pflags == 0x40:
rgb = True
elif pflags == 0x41:
rgb = True
has_alpha = True
else:
util.LogDebug("")
util.LogDebug("Invalid texture. pflags = " + str(pflags))
return 0, 0, 0, b'', 0, [], 0, []
format_ = 0
if compressed:
compSel = [0, 1, 2, 3]
if fourcc == b'DXT1':
format_ = 0x42
bpp = 8
elif fourcc == b'DXT3':
format_ = 0x43
bpp = 16
elif fourcc == b'DXT5':
format_ = 0x44
bpp = 16
elif fourcc in [b'BC4U', b'ATI1']:
format_ = 0x49
bpp = 8
elif fourcc == b'BC4S':
format_ = 0x4a
bpp = 8
elif fourcc in [b'BC5U', b'ATI2']:
format_ = 0x4b
bpp = 16
elif fourcc == b'BC5S':
format_ = 0x4c
bpp = 16
size = ((width + 3) >> 2) * ((height + 3) >> 2) * bpp
else:
if luminance:
if has_alpha:
if channel0 in a8l8_masks and channel1 in a8l8_masks and channel2 in a8l8_masks and channel3 in a8l8_masks and bpp == 2:
format_ = 0xd
compSel = [
a8l8_masks[channel0], a8l8_masks[channel1],
a8l8_masks[channel2], a8l8_masks[channel3]
]
else:
if channel0 in l8_masks and channel1 in l8_masks and channel2 in l8_masks and channel3 in l8_masks and bpp == 1:
format_ = 1
compSel = [
l8_masks[channel0], l8_masks[channel1],
l8_masks[channel2], l8_masks[channel3]
]
elif rgb:
if has_alpha:
if bpp == 4:
if channel0 in abgr8_masks and channel1 in abgr8_masks and channel2 in abgr8_masks and channel3 in abgr8_masks:
format_ = 0x38 if SRGB else 0x25
compSel = [
abgr8_masks[channel0], abgr8_masks[channel1],
abgr8_masks[channel2], abgr8_masks[channel3]
]
elif channel0 in a2rgb10_masks and channel1 in a2rgb10_masks and channel2 in a2rgb10_masks and channel3 in a2rgb10_masks:
format_ = 0x3d
compSel = [
a2rgb10_masks[channel0], a2rgb10_masks[channel1],
a2rgb10_masks[channel2], a2rgb10_masks[channel3]
]
elif bpp == 2:
if channel0 in a1bgr5_masks and channel1 in a1bgr5_masks and channel2 in a1bgr5_masks and channel3 in a1bgr5_masks:
format_ = 0x3b
compSel = [
a1bgr5_masks[channel0], a1bgr5_masks[channel1],
a1bgr5_masks[channel2], a1bgr5_masks[channel3]
]
elif channel0 in abgr4_masks and channel1 in abgr4_masks and channel2 in abgr4_masks and channel3 in abgr4_masks:
format_ = 0x39
compSel = [
abgr4_masks[channel0], abgr4_masks[channel1],
abgr4_masks[channel2], abgr4_masks[channel3]
]
else:
if channel0 in bgr8_masks and channel1 in bgr8_masks and channel2 in bgr8_masks and channel3 == 0 and bpp == 3: # Kinda not looking good if you ask me
format_ = 0x38 if SRGB else 0x25
compSel = [
bgr8_masks[channel0], bgr8_masks[channel1],
bgr8_masks[channel2], 3
]
if channel0 in bgr565_masks and channel1 in bgr565_masks and channel2 in bgr565_masks and channel3 in bgr565_masks and bpp == 2:
format_ = 0x3c
compSel = [
bgr565_masks[channel0], bgr565_masks[channel1],
bgr565_masks[channel2], bgr565_masks[channel3]
]
size = width * height * bpp
if caps == 0x401008:
numMips = struct.unpack("<I", inb[28:32])[0] - 1
mipSize = get_mipSize(width, height, bpp, numMips, compressed)
else:
numMips = 0
mipSize = 0
if len(inb) < 0x80 + size + mipSize:
util.LogDebug("")
util.LogDebug(f + " is not a valid DDS file!")
return 0, 0, 0, b'', 0, [], 0, []
if format_ == 0:
util.LogDebug("")
util.LogDebug("Unsupported DDS format!")
return 0, 0, 0, b'', 0, [], 0, []
data = inb[0x80:0x80 + size + mipSize]
if format_ in [0x25, 0x38] and bpp == 3:
data = form_conv.rgb8torgbx8(data)
bpp += 1
size = width * height * bpp
return width, height, format_, fourcc, size, compSel, numMips, data
def ProcessBatch(self):
util.LogDebug("JM Process Batch {}".format(len(self.m_JobsBacklog)))
while len(self.m_RunningJobs) > 0 or len(self.m_JobsBacklog) > 0:
self.Update()
util.LogDebug("JM Batch Processed")
def ConvertPath(mod_name, target):
script_path = util.GetScriptPath()
util.LogInfo("Convert Path")
util.LogDebug("This is the target: " + target)
util.LogDebug("This is the mod name " + mod_name)
has_havoc = util.HasHavokBPP()
do_meshes = \
toolkit_config.get_bool_setting("Meshes", "RemoveEditorMarker") or \
toolkit_config.get_bool_setting("Meshes", "PrettySortBlocks") or \
toolkit_config.get_bool_setting("Meshes", "TrimTexturesPath") or \
toolkit_config.get_bool_setting("Meshes", "OptimizeForSSE")
NoConversion = {}
WarnConversion = {}
ConvertListDDS = []
ConvertListHKX = []
ConvertListHKX64 = []
ConvertListTXT = []
ConvertListSound = []
ConvertListMesh = []
for root, subdirs, files in os.walk(target):
if root != target:
util.LogDebug("Walking folder " + root)
for filename in files:
#util.LogDebug("filename: {}".format(filename))
if filename.lower().endswith(".dds"):
file_path = os.path.join(root, filename)
ddsChecked = check_dds.CheckDDS(file_path, file_path)
if ddsChecked == check_dds.PC:
ConvertListDDS.append(file_path)
elif ddsChecked == check_dds.NX:
if 'DDS' not in NoConversion:
NoConversion['DDS'] = 0
NoConversion['DDS'] += 1
else:
if 'DDS' not in WarnConversion:
WarnConversion['DDS'] = []
WarnConversion['DDS'].append(
(filename, "Unknown DDS format"))
elif filename.lower().endswith(".hkx"):
file_path = os.path.join(root, filename)
hkxChecked = check_hkx.CheckHKX(file_path, file_path)
if hkxChecked == check_hkx.PC_32:
ConvertListHKX.append(file_path)
elif hkxChecked == check_hkx.PC_XML:
ConvertListHKX.append(file_path)
elif hkxChecked == check_hkx.PC_64:
# if root.lower().find("behaviors") > 0 or filename.lower().find("skeleton") > 0:
# if 'HKX' not in WarnConversion:
# WarnConversion['HKX'] = []
# WarnConversion['HKX'].append( (filename, "SSE hkx animation") )
# else:
ConvertListHKX64.append(file_path)
elif hkxChecked == check_hkx.NX_64:
if 'HKX' not in NoConversion:
NoConversion['HKX'] = 0
NoConversion['HKX'] += 1
else:
if 'HKX' not in WarnConversion:
WarnConversion['HKX'] = []
WarnConversion['HKX'].append(
(filename, "Unknown hkx animation format"))
elif filename.lower().startswith(
"translate_") and filename.lower().endswith(".txt"):
file_path = os.path.join(root, filename)
ConvertListTXT.append(file_path)
elif filename.lower().endswith(".xwm") or filename.lower(
).endswith(".fuz") or filename.lower().endswith(".wav"):
file_path = os.path.join(root, filename[:-4])
if not file_path in ConvertListSound:
ConvertListSound.append(file_path)
elif filename.lower().endswith(".nif"):
file_path = os.path.join(root, filename)
ConvertListMesh.append(file_path)
for fileType in NoConversion:
util.LogInfo("Found {} {} files that are already in NX format".format(
NoConversion[fileType], fileType))
for fileType in WarnConversion:
fileTypeWarnings = WarnConversion[fileType]
for i in range(len(fileTypeWarnings)):
util.LogInfo("Warning, cannot convert {} because <{}>".format(
fileTypeWarnings[i][0], fileTypeWarnings[i][1]))
util.LogInfo("Found {} dds files to convert".format(len(ConvertListDDS)))
if has_havoc:
util.LogInfo("Found {} 32-bit hkx files to convert".format(
len(ConvertListHKX)))
else:
util.LogInfo(
"Found {} 32-bit hkx files that won't convert as Havoc utility wasn't found."
.format(len(ConvertListHKX)))
util.LogInfo("Found {} 64-bit hkx files to convert".format(
len(ConvertListHKX64)))
util.LogInfo("Found {} txt files to convert".format(len(ConvertListTXT)))
util.LogInfo("Found {} sound files to convert".format(
len(ConvertListSound)))
if do_meshes:
util.LogInfo("Found {} mesh files to convert".format(
len(ConvertListMesh)))
else:
util.LogInfo("Found {} mesh files but won't touch them.".format(
len(ConvertListMesh)))
'''
def LogProgress(convertList, convertFn, name):
if len(convertList) > 0:
failedCount = 0
for i in range(len(convertList)):
file_path = convertList[i]
success = convertFn(target, file_path)
if not success:
failedCount += 1
sys.stdout.write("Converted {}/{} {} ({}) failed. \r".format(i+1, len(convertList), name, failedCount))
sys.stdout.flush()
sys.stdout.write("\n")
'''
def LogProgress(convertList, fnName, convertFn, name, threadSetting):
if len(convertList) > 0:
failedCount = 0
maxThreads = toolkit_config.get_int_setting(
"Performance", threadSetting)
jm = job_manager.JobManager(maxThreads)
convertedCount = 0
processedCount = 0
totalCount = len(convertList)
def cb(success):
nonlocal processedCount, convertedCount, failedCount
processedCount += 1
if success:
convertedCount += 1
else:
failedCount += 1
sys.stdout.write(
"{} Processed {}/{} ({}/{}) success/failure. \r".format(
name, processedCount, totalCount, convertedCount,
failedCount))
sys.stdout.flush()
for i in range(len(convertList)):
file_path = convertList[i]
job = job_manager.Job(cb, fnName, convertFn, target.lower(),
file_path.lower())
jm.AddJob(job)
jm.ProcessBatch()
sys.stdout.write(
"{} Processing Complete: {}/{} ({}/{}) success/failure. \r".
format(name, processedCount, totalCount, convertedCount,
failedCount))
sys.stdout.write("\n")
if processedCount != totalCount:
sys.stdout.write("Not all were processed.\n")
sys.stdout.flush()
LogProgress(ConvertListDDS, "ConvertDDS", convert_dds.ConvertDDS, "DDS",
"MaxTextureThreads")
if has_havoc:
LogProgress(ConvertListHKX, "ConvertHKX", convert_hkx.ConvertHKX,
"HKX 32-bit", "MaxAnimationThreads")
LogProgress(ConvertListHKX64, "ConvertHKX64", convert_hkx64.ConvertHKX64,
"HKX 64-bit", "MaxAnimationThreads")
LogProgress(ConvertListTXT, "ConvertTXT", convert_txt.ConvertTXT, "TXT",
"MaxOtherThreads")
LogProgress(ConvertListSound, "ConvertSound", convert_sound.ConvertSound,
"Sounds", "MaxSoundThreads")
if do_meshes:
LogProgress(ConvertListMesh, "ConvertMesh", convert_nif.ConvertNIF,
"Meshes", "MaxMeshThreads")
def main(arguments):
util.LogDebug("XTX Extractor v0.1")
util.LogDebug("(C) 2017 Stella/AboodXD")
input_ = arguments[-1]
if not (input_.endswith('.xtx') or input_.endswith('.dds')):
printInfo()
toXTX = False
if input_.endswith('.dds'):
toXTX = True
if "-o" in arguments:
output_ = arguments[arguments.index("-o") + 1]
else:
output_ = os.path.splitext(input_)[0] + (".xtx" if toXTX else ".dds")
if toXTX:
if "-SRGB" in arguments:
SRGB = int(arguments[arguments.index("-SRGB") + 1], 0)
else:
SRGB = 0
multi = False
if "-multi" in arguments:
multi = True
numImages = int(arguments[arguments.index("-multi") + 1], 0)
if SRGB > 1:
printInfo()
if "-o" not in arguments and "-multi" in arguments:
output_ = output_[:-5] + ".xtx"
with open(output_, "wb+") as output:
head_struct = NvHeader()
head = head_struct.pack(0x4E764644, 16, 1, 1)
output.write(head)
numBlocks = 0
if multi:
input_ = input_[:-5]
for i in range(numImages):
util.LogDebug("")
util.LogDebug('Converting: ' + input_ + str(i) + ".dds")
data, numBlocks = writeNv(input_ + str(i) + ".dds", SRGB,
i, numImages, numBlocks)
output.write(data)
else:
util.LogDebug("")
util.LogDebug('Converting: ' + input_)
data, numBlocks = writeNv(input_, SRGB, 0, 1, numBlocks)
output.write(data)
block_head_struct = NvBlockHeader()
eof_blk_head = block_head_struct.pack(0x4E764248, 0x24, 0x18, 0x24,
5, numBlocks, 0)
output.write(eof_blk_head)
output.write(
b'\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
)
else:
util.LogDebug("")
util.LogDebug('Converting: ' + input_)
with open(input_, "rb") as inf:
inb = inf.read()
nv = readNv(inb)
for i in range(nv.numImages):
util.LogDebug("")
util.LogDebug("// ----- NvTextureHeader Info ----- ")
util.LogDebug(" imageSize = " + str(nv.imageSize[i]))
util.LogDebug(" alignment = " + str(nv.alignment[i]))
util.LogDebug(" width = " + str(nv.width[i]))
util.LogDebug(" height = " + str(nv.height[i]))
util.LogDebug(" depth = " + str(nv.depth[i]))
util.LogDebug(" target = " + str(nv.target[i]))
if nv.format[i] in formats:
util.LogDebug(" format = " + formats[nv.format[i]])
else:
util.LogDebug(" format = " + hex(nv.format[i]))
util.LogDebug(" numMips = " + str(nv.numMips[i]))
util.LogDebug(" sliceSize = " + str(nv.sliceSize[i]))
if nv.format[i] in formats:
bpp = nv.bpp[i]
util.LogDebug("")
util.LogDebug(" bits per pixel = " + str(bpp * 8))
util.LogDebug(" bytes per pixel = " + str(bpp))
if nv.numImages > 1:
output_ = os.path.splitext(input_)[0] + str(i) + ".dds"
hdr, result = get_deswizzled_data(i, nv)
if hdr == b'' or result == []:
pass
else:
with open(output_, "wb+") as output:
output.write(hdr)
for data in result:
output.write(data)
util.LogDebug('')
util.LogDebug('Finished converting: ' + input_)
def printInfo():
util.LogDebug("")
util.LogDebug("Usage:")
util.LogDebug(" xtx_extract [option...] input")
util.LogDebug("")
util.LogDebug("Options:")
util.LogDebug(
" -o <output> Output file, if not specified, the output file will have the same name as the intput file"
)
util.LogDebug(
" Will be ignored if the XTX has multiple images"
)
util.LogDebug("")
util.LogDebug("DDS to XTX options:")
util.LogDebug(
" -SRGB <n> 1 if the desired destination format is SRGB, else 0 (0 is the default)"
)
util.LogDebug(
" -multi <numImages> number of images to pack into the XTX file (input file must be the first image, 1 is the default)"
)
util.LogDebug("")
util.LogDebug("Supported formats:")
util.LogDebug(" - NVN_FORMAT_RGBA8")
util.LogDebug(" - NVN_FORMAT_RGBA8_SRGB")
util.LogDebug(" - NVN_FORMAT_RGB10A2")
util.LogDebug(" - NVN_FORMAT_RGB565")
util.LogDebug(" - NVN_FORMAT_RGB5A1")
util.LogDebug(" - NVN_FORMAT_RGBA4")
util.LogDebug(" - NVN_FORMAT_R8")
util.LogDebug(" - NVN_FORMAT_RG8")
util.LogDebug(" - DXT1")
util.LogDebug(" - DXT3")
util.LogDebug(" - DXT5")
util.LogDebug(" - BC4U")
util.LogDebug(" - BC4S")
util.LogDebug(" - BC5U")
util.LogDebug(" - BC5S")
util.LogDebug("")
util.LogDebug("Exiting in 5 seconds...")
time.sleep(5)
sys.exit(1)