def blockCompressXci(filePath, compressionLevel, blockSizeExponent, outputDir,
threads):
filePath = filePath.resolve()
container = factory(filePath)
container.open(str(filePath), 'rb')
secureIn = container.hfs0['secure']
xczPath = outputDir.joinpath(filePath.stem + '.xcz')
Print.info('Block compressing (level {0}) {1} -> {2}'.format(
compressionLevel, filePath, xczPath))
try:
with Xci.XciStream(
str(xczPath), originalXciPath=filePath
) as xci: # need filepath to copy XCI container settings
with Hfs0.Hfs0Stream(xci.hfs0.add('secure', 0),
xci.f.tell()) as secureOut:
blockCompressContainer(secureIn, secureOut, compressionLevel,
blockSizeExponent, threads)
xci.hfs0.resize('secure', secureOut.actualSize)
except BaseException as ex:
if not ex is KeyboardInterrupt:
Print.error(format_exc())
if xczPath.is_file():
xczPath.unlink()
container.close()
return xczPath
def ExtractTitleIDAndVersion(gamePath, args = None):
titleId = ""
version = -1
gameName = Path(gamePath).name
titleIdResult = search(r'0100[0-9A-Fa-f]{12}', gameName)
if args == None or not args.alwaysParseCnmt:
if titleIdResult:
titleId = titleIdResult.group().upper()
versionResult = search(r'\[v\d+\]', gameName)
if versionResult:
version = int(versionResult.group()[2:-1])
if titleId != "" and version > -1 and version%65536 == 0:
return(titleId, version)
elif args != None and not args.parseCnmt:
return None
gamePath = Path(gamePath).resolve()
container = factory(gamePath)
container.open(str(gamePath), 'rb')
if isXciXcz(gamePath):
container = container.hfs0['secure']
try:
for nspf in container:
if isinstance(nspf, Nca.Nca) and nspf.header.contentType == Type.Content.META:
for section in nspf:
if isinstance(section, Pfs0.Pfs0):
Cnmt = section.getCnmt()
titleId = Cnmt.titleId.upper()
version = Cnmt.version
finally:
container.close()
if titleId != "" and version > -1 and version%65536 == 0:
return(titleId, version)
return None
def __decompressNsz(filePath, outputDir, write, raiseVerificationException,
statusReportInfo, pleaseNoPrint):
fileHashes = FileExistingChecks.ExtractHashes(filePath)
container = factory(filePath)
container.open(str(filePath), 'rb')
try:
if write:
filename = changeExtension(filePath, '.nsp')
outPath = filename if outputDir == None else str(
Path(outputDir).joinpath(filename))
Print.info('Decompressing %s -> %s' % (filePath, outPath),
pleaseNoPrint)
with Pfs0.Pfs0Stream(outPath) as nsp:
__decompressContainer(container, nsp, fileHashes, write,
raiseVerificationException,
statusReportInfo, pleaseNoPrint)
else:
__decompressContainer(container, None, fileHashes, write,
raiseVerificationException, statusReportInfo,
pleaseNoPrint)
except BaseException:
raise
finally:
container.close()
def __decompressXcz(filePath, outputDir, write, raiseVerificationException,
statusReportInfo, pleaseNoPrint):
fileHashes = FileExistingChecks.ExtractHashes(filePath)
container = factory(filePath)
container.open(str(filePath), 'rb')
secureIn = container.hfs0['secure']
if write:
filename = changeExtension(filePath, '.xci')
outPath = filename if outputDir == None else str(
Path(outputDir).joinpath(filename))
Print.info('Decompressing %s -> %s' % (filePath, outPath),
pleaseNoPrint)
with Xci.XciStream(
outPath, originalXciPath=filePath
) as xci: # need filepath to copy XCI container settings
with Hfs0.Hfs0Stream(xci.hfs0.add('secure', 0, pleaseNoPrint),
xci.f.tell()) as secureOut:
__decompressContainer(secureIn, secureOut, fileHashes, write,
raiseVerificationException,
statusReportInfo, pleaseNoPrint)
xci.hfs0.resize('secure', secureOut.actualSize)
else:
__decompressContainer(secureIn, None, fileHashes, write,
raiseVerificationException, statusReportInfo,
pleaseNoPrint)
container.close()
def decompress(filePath, outputDir, statusReportInfo, pleaseNoPrint = None):
if isNspNsz(filePath):
__decompressNsz(filePath, outputDir, True, False, statusReportInfo, pleaseNoPrint)
elif isXciXcz(filePath):
__decompressXcz(filePath, outputDir, True, False, statusReportInfo, pleaseNoPrint)
elif isCompressedGameFile(filePath):
filename = changeExtension(filePath, '.nca')
outPath = filename if outputDir == None else str(Path(outputDir).joinpath(filename))
Print.info('Decompressing %s -> %s' % (filePath, outPath), pleaseNoPrint)
try:
inFile = factory(filePath)
inFile.open(str(filePath), 'rb')
with open(outPath, 'wb') as outFile:
written, hexHash = __decompressNcz(inFile, outFile, statusReportInfo, pleaseNoPrint)
fileNameHash = Path(filePath).stem.lower()
if hexHash[:32] == fileNameHash:
Print.info('[VERIFIED] {0}'.format(filename), pleaseNoPrint)
else:
Print.info('[MISMATCH] Filename startes with {0} but {1} was expected - hash verified failed!'.format(fileNameHash, hexHash[:32]), pleaseNoPrint)
except BaseException as ex:
if not ex is KeyboardInterrupt:
Print.error(format_exc())
if Path(outPath).is_file():
Path(outPath).unlink()
finally:
inFile.close()
else:
raise NotImplementedError("Can't decompress {0} as that file format isn't implemented!".format(filePath))
def ExtractHashes(gamePath): fileHashes = set() gamePath = gamePath.resolve() container = factory(gamePath) container.open(str(gamePath), 'rb') if isXciXcz(gamePath): container = container.hfs0['secure'] try: for nspf in container: if isinstance(nspf, Nca.Nca) and nspf.header.contentType == Type.Content.META: for section in nspf: if isinstance(section, Pfs0.Pfs0): Cnmt = section.getCnmt() for entry in Cnmt.contentEntries: fileHashes.add(entry.hash.hex()) finally: container.close() return fileHashes
def blockCompressNsp(filePath, compressionLevel , blockSizeExponent, outputDir, threads):
filePath = filePath.resolve()
container = factory(filePath)
container.open(str(filePath), 'rb')
nszPath = outputDir.joinpath(filePath.stem + '.nsz')
Print.info('Block compressing (level {0}) {1} -> {2}'.format(compressionLevel, filePath, nszPath))
try:
with Pfs0.Pfs0Stream(str(nszPath)) as nsp:
blockCompressContainer(container, nsp, compressionLevel, blockSizeExponent, threads)
except BaseException as ex:
if not ex is KeyboardInterrupt:
Print.error(format_exc())
if nszPath.is_file():
nszPath.unlink()
container.close()
return nszPath
def solidCompressNsp(filePath, compressionLevel, outputDir, threads,
stusReport, id, pleaseNoPrint):
filePath = filePath.resolve()
container = factory(filePath)
container.open(str(filePath), 'rb')
nszPath = outputDir.joinpath(filePath.stem + '.nsz')
Print.info(
'Solid compressing (level {0}) {1} -> {2}'.format(
compressionLevel, filePath, nszPath), pleaseNoPrint)
try:
with Pfs0.Pfs0Stream(str(nszPath)) as nsp:
processContainer(container, nsp, compressionLevel, threads,
stusReport, id, pleaseNoPrint)
except BaseException as ex:
if not ex is KeyboardInterrupt:
Print.error(format_exc())
if nszPath.is_file():
nszPath.unlink()
container.close()
return nszPath
def main():
global err
try:
if len(argv) > 1:
args = ParseArguments.parse()
else:
try:
from nsz.gui.NSZ_GUI import GUI
except ImportError:
Print.error("Failed to import the GUI - is it installed?")
return
args = GUI().run()
if args == None:
Print.info("Done!")
return
if args.output:
outfolderToPharse = args.output
if not outfolderToPharse.endswith('/') and not outfolderToPharse.endswith('\\'):
outfolderToPharse += "/"
if not Path(outfolderToPharse).is_dir():
Print.error('Error: Output directory "{0}" does not exist!'.format(args.output))
return
outfolder = Path(outfolderToPharse).resolve() if args.output else Path('.').resolve()
Print.info('')
Print.info(' NSZ v3.1 ,;:;;,')
Print.info(' ;;;;;')
Print.info(' .=\', ;:;;:,')
Print.info(' /_\', "=. \';:;:;')
Print.info(' @=:__, \,;:;:\'')
Print.info(' _(\.= ;:;;\'')
Print.info(' `"_( _/="`')
Print.info(' `"\'')
Print.info('')
barManager = enlighten.get_manager()
poolManager = Manager()
statusReport = poolManager.list()
readyForWork = Counter(0)
pleaseNoPrint = Counter(0)
pleaseKillYourself = Counter(0)
pool = []
work = poolManager.Queue()
amountOfTastkQueued = Counter(0)
if args.titlekeys:
extractTitlekeys(args.file)
if args.extract:
for f_str in args.file:
for filePath in expandFiles(Path(f_str)):
filePath_str = str(filePath)
Print.info('Extracting "{0}" to {1}'.format(filePath_str, outfolder))
dir = outfolder.joinpath(filePath.stem)
container = factory(filePath)
container.open(filePath_str, 'rb')
if isXciXcz(filePath):
for hfs0 in container.hfs0:
secureIn = hfs0
secureIn.unpack(dir.joinpath(hfs0._path), args.extractregex)
else:
container.unpack(dir, args.extractregex)
container.close()
if args.create:
Print.info('Creating "{0}"'.format(args.create))
nsp = Nsp.Nsp(None, None)
nsp.path = args.create
nsp.pack(args.file)
if args.C:
targetDictNsz = CreateTargetDict(outfolder, args.parseCnmt, ".nsz")
targetDictXcz = CreateTargetDict(outfolder, args.parseCnmt, ".xcz")
sourceFileToDelete = []
for f_str in args.file:
for filePath in expandFiles(Path(f_str)):
if not isUncompressedGame(filePath):
continue
try:
if filePath.suffix == '.nsp':
if not AllowedToWriteOutfile(filePath, ".nsz", targetDictNsz, args.rm_old_version, args.overwrite, args.parseCnmt):
continue
elif filePath.suffix == '.xci':
if not AllowedToWriteOutfile(filePath, ".xcz", targetDictXcz, args.rm_old_version, args.overwrite, args.parseCnmt):
continue
compress(filePath, outfolder, args, work, amountOfTastkQueued)
if args.rm_source:
sourceFileToDelete.append(filePath)
except KeyboardInterrupt:
raise
except BaseException as e:
Print.error('Error when compressing file: %s' % filePath)
err.append({"filename":filePath,"error":format_exc() })
print_exc()
bars = []
compressedSubBars = []
BAR_FMT = u'{desc}{desc_pad}{percentage:3.0f}%|{bar}| {count:{len_total}d}/{total:d} {unit} [{elapsed}<{eta}, {rate:.2f}{unit_pad}{unit}/s]'
parallelTasks = min(args.multi, amountOfTastkQueued.value())
if parallelTasks < 0:
parallelTasks = 4
for i in range(parallelTasks):
statusReport.append([0, 0, 100])
p = Process(target=solidCompressTask, args=(work, statusReport, readyForWork, pleaseNoPrint, pleaseKillYourself, i))
p.start()
pool.append(p)
for i in range(parallelTasks):
bar = barManager.counter(total=100, desc='Compressing', unit='MiB', color='cyan', bar_format=BAR_FMT)
compressedSubBars.append(bar.add_subcounter('green'))
bars.append(bar)
#Ensures that all threads are started and compleaded before being requested to quit
while readyForWork.value() < parallelTasks:
sleep(0.2)
if pleaseNoPrint.value() > 0:
continue
pleaseNoPrint.increment()
for i in range(parallelTasks):
compressedRead, compressedWritten, total = statusReport[i]
if bars[i].total != total:
bars[i].total = total//1048576
bars[i].count = compressedRead//1048576
compressedSubBars[i].count = compressedWritten//1048576
bars[i].refresh()
pleaseNoPrint.decrement()
pleaseKillYourself.increment()
for i in range(readyForWork.value()):
work.put(None)
while readyForWork.value() > 0:
sleep(0.02)
for i in range(parallelTasks):
bars[i].close(clear=True)
#barManager.stop() #We aren’t using stop because of it printing garbage to the console.
for filePath in sourceFileToDelete:
delete_source_file(filePath)
if args.D:
targetDictNsz = CreateTargetDict(outfolder, args.parseCnmt, ".nsp")
targetDictXcz = CreateTargetDict(outfolder, args.parseCnmt, ".xci")
for f_str in args.file:
for filePath in expandFiles(Path(f_str)):
if not isCompressedGame(filePath) and not isCompressedGameFile(filePath):
continue
try:
if filePath.suffix == '.nsz':
if not AllowedToWriteOutfile(filePath, ".nsp", targetDictNsz, args.rm_old_version, args.overwrite, args.parseCnmt):
continue
elif filePath.suffix == '.xcz':
if not AllowedToWriteOutfile(filePath, ".xci", targetDictXcz, args.rm_old_version, args.overwrite, args.parseCnmt):
continue
elif filePath.suffix == '.ncz':
outfile = Path(changeExtension(outfolder.joinpath(filePath.name), ".nca"))
if not args.overwrite and outfile.is_file():
Print.info('{0} with the same file name already exists in the output directory.\n'\
'If you want to overwrite it use the -w parameter!'.format(outfile.name))
continue
decompress(filePath, outfolder)
if args.rm_source:
delete_source_file(filePath)
except KeyboardInterrupt:
raise
except BaseException as e:
Print.error('Error when decompressing file: {0}'.format(filePath))
err.append({"filename":filePath, "error":format_exc()})
print_exc()
if args.info:
for f_str in args.file:
for filePath in expandFiles(Path(f_str)):
filePath_str = str(filePath)
Print.info(filePath_str)
f = factory(filePath)
f.open(filePath_str, 'r+b')
f.printInfo(args.depth+1)
f.close()
if args.verify and not args.C and not args.D:
for f_str in args.file:
for filePath in expandFiles(Path(f_str)):
try:
if isGame(filePath):
Print.info("[VERIFY {0}] {1}".format(getExtensionName(filePath), filePath.name))
verify(filePath, False)
except KeyboardInterrupt:
raise
except BaseException as e:
Print.error('Error when verifying file: {0}'.format(filePath))
err.append({"filename":filePath,"error":format_exc()})
print_exc()
if len(argv) == 1:
pass
except KeyboardInterrupt:
Print.info('Keyboard exception')
except BaseException as e:
Print.info('nut exception: {0}'.format(str(e)))
raise
if err:
Print.info('\n\033[93m\033[1mSummary of errors which occurred while processing files:')
for e in err:
Print.info('\033[0mError while processing {0}'.format(e["filename"]))
Print.info(e["error"])
Print.info('Done!')
_exit(0)
def extractTitlekeys(argsFile):
titlekeysDict = {}
if Path("titlekeys.txt").is_file():
with open("titlekeys.txt", "r", encoding="utf-8") as titlekeysFile:
for line in titlekeysFile:
(rightsId, titleKey, name) = line.rstrip().split('|')
titlekeysDict[rightsId[0:16]] = (rightsId, titleKey, name)
#Print.info("Read: {0}|{1}|{2}".format(rightsId, titleKey, name))
for f_str in argsFile:
for filePath in expandFiles(Path(f_str)):
if not isNspNsz(filePath):
continue
f = factory(filePath)
f.open(str(filePath), 'rb')
ticket = f.ticket()
rightsId = format(ticket.getRightsId(), 'x').zfill(32)
titleId = rightsId[0:16]
if not titleId in titlekeysDict:
titleKey = format(ticket.getTitleKeyBlock(), 'x').zfill(32)
titlekeysDict[titleId] = (rightsId, titleKey, filePath.stem)
Print.info("Found: {0}|{1}|{2}".format(rightsId, titleKey, filePath.stem))
else:
Print.info("Skipped already existing {0}".format(rightsId))
f.close()
Print.info("\ntitlekeys.txt:")
with open('titlekeys.txt', 'w', encoding="utf-8") as titlekeysFile:
for titleId in sorted(titlekeysDict.keys()):
(rightsId, titleKey, name) = titlekeysDict[titleId]
titleDBLine = "{0}|{1}|{2}".format(rightsId, titleKey, name)
titlekeysFile.write(titleDBLine + '\n')
Print.info(titleDBLine)
if not Path("./titledb/").is_dir():
return
Print.info("\n\ntitledb:")
Print.info("========")
titleDbPath = Path("titledb").resolve()
excludeJson = {"cheats.json", "cnmts.json", "languages.json", "ncas.json", "versions.json"}
for filePath in expandFiles(titleDbPath):
if filePath.suffix == '.json':
fileName = filePath.name
if fileName in excludeJson:
continue
saveTrigger = False
Print.info("Reading {0}".format(fileName))
with open(str(filePath)) as json_file:
data = json.load(json_file)
for key, value in data.items():
titleId = value["id"]
if titleId == None:
continue
if titleId in titlekeysDict:
(rightsId, titleKey, name) = titlekeysDict[titleId]
if value["rightsId"] == None:
value["rightsId"] = rightsId
saveTrigger=True
#elif value["rightsId"] != rightsId:
# Print.info("Warn: {0} != {1}".format(value["rightsId"], rightsId))
if value["key"] == None:
Print.info("{0}: Writing key to {1}".format(fileName, titleId))
value["key"] = titleKey
saveTrigger=True
if saveTrigger == True:
Print.info("Saving {0}".format(fileName))
with open(str(filePath), 'w') as outfile:
json.dump(data, outfile, indent=4, sort_keys=True)
Print.info("{0} saved!".format(fileName))
Print.info("")
