def __decompressNsz(filePath, outputDir, write, raiseVerificationException,
statusReportInfo):
fileHashes = [] # FileExistingChecks.ExtractHashes(filePath)
container = factory(filePath)
container.open(str(filePath), 'rb')
for f in container:
if isCompressedGameFile(f._path):
fileHashes.append(f._path.split('.')[0])
if write:
filename = changeExtension(filePath, '.nsp')
outPath = filename if outputDir is None else os.path.join(
outputDir, os.path.basename(filename))
Print.info('Decompressing %s -> %s' % (filePath, outPath))
if Config.dryRun:
return outPath
with Pfs0Stream(outPath) as nsp:
__decompressContainer(container, nsp, fileHashes, write,
raiseVerificationException, statusReportInfo)
return outPath
else:
__decompressContainer(container, None, fileHashes, write,
raiseVerificationException, statusReportInfo)
container.close()
return None
def ExtractTitleIDAndVersion(gamePath, parseCnmt):
titleId = ""
version = -1
gameName = Path(gamePath).name
titleIdResult = search(r'0100[0-9A-Fa-f]{12}', gameName)
if titleIdResult:
titleId = titleIdResult.group().upper()
versionResult = search(r'\[v\d+\]', gameName)
if versionResult:
version = int(versionResult.group()[2:-1])
if titleId != "" and version > -1 and version % 65536 == 0:
return (titleId, version)
elif not parseCnmt:
return None
gamePath = Path(gamePath).resolve()
container = factory(gamePath)
container.open(str(gamePath), 'rb')
if isXciXcz(gamePath):
container = container.hfs0['secure']
try:
for nspf in container:
if isinstance(
nspf,
Nca.Nca) and nspf.header.contentType == Type.Content.META:
for section in nspf:
if isinstance(section, Pfs0.Pfs0):
Cnmt = section.getCnmt()
titleId = Cnmt.titleId.upper()
version = Cnmt.version
finally:
container.close()
if titleId != "" and version > -1 and version % 65536 == 0:
return (titleId, version)
return None
def decompress(filePath, outputDir, statusReportInfo, pleaseNoPrint = None):
if isNspNsz(filePath):
__decompressNsz(filePath, outputDir, True, False, statusReportInfo, pleaseNoPrint)
elif isXciXcz(filePath):
__decompressXcz(filePath, outputDir, True, False, statusReportInfo, pleaseNoPrint)
elif isCompressedGameFile(filePath):
filename = changeExtension(filePath, '.nca')
outPath = filename if outputDir == None else str(Path(outputDir).joinpath(filename))
Print.info('Decompressing %s -> %s' % (filePath, outPath), pleaseNoPrint)
container = factory(filePath)
container.open(filePath, 'rb')
try:
with open(outPath, 'wb') as outFile:
written, hexHash = __decompressNcz(container, outFile)
except BaseException as ex:
if not ex is KeyboardInterrupt:
Print.error(format_exc())
if outFile.is_file():
outFile.unlink()
finally:
container.close()
fileNameHash = Path(filePath).stem.lower()
if hexHash[:32] == fileNameHash:
Print.info('[VERIFIED] {0}'.format(filename), pleaseNoPrint)
else:
Print.info('[MISMATCH] Filename startes with {0} but {1} was expected - hash verified failed!'.format(fileNameHash, hexHash[:32]), pleaseNoPrint)
else:
raise NotImplementedError("Can't decompress {0} as that file format isn't implemented!".format(filePath))
def __decompressXcz(filePath, outputDir, write, raiseVerificationException,
statusReportInfo, pleaseNoPrint):
fileHashes = FileExistingChecks.ExtractHashes(filePath)
container = factory(filePath)
container.open(filePath, 'rb')
secureIn = container.hfs0['secure']
if write:
filename = changeExtension(filePath, '.xci')
outPath = filename if outputDir == None else str(
Path(outputDir).joinpath(filename))
Print.info('Decompressing %s -> %s' % (filePath, outPath),
pleaseNoPrint)
with Xci.XciStream(
outPath, originalXciPath=filePath
) as xci: # need filepath to copy XCI container settings
with Hfs0.Hfs0Stream(xci.hfs0.add('secure', 0, pleaseNoPrint),
xci.f.tell()) as secureOut:
__decompressContainer(secureIn, secureOut, fileHashes, write,
raiseVerificationException,
statusReportInfo, pleaseNoPrint)
xci.hfs0.resize('secure', secureOut.actualSize)
else:
__decompressContainer(secureIn, None, fileHashes, write,
raiseVerificationException, statusReportInfo,
pleaseNoPrint)
container.close()
def solidCompressXci(filePath, compressionLevel, outputDir, threads,
stusReport, id, pleaseNoPrint):
filePath = filePath.resolve()
container = factory(filePath)
container.open(str(filePath), 'rb')
secureIn = container.hfs0['secure']
xczPath = outputDir.joinpath(filePath.stem + '.xcz')
Print.info(
'Solid compressing (level {0}) {1} -> {2}'.format(
compressionLevel, filePath, xczPath), pleaseNoPrint)
try:
with Xci.XciStream(
str(xczPath), originalXciPath=filePath
) as xci: # need filepath to copy XCI container settings
with Hfs0.Hfs0Stream(xci.hfs0.add('secure', 0, pleaseNoPrint),
xci.f.tell()) as secureOut:
processContainer(secureIn, secureOut, compressionLevel,
threads, stusReport, id, pleaseNoPrint)
xci.hfs0.resize('secure', secureOut.actualSize)
except BaseException as ex:
if not ex is KeyboardInterrupt:
Print.error(format_exc())
if xczPath.is_file():
xczPath.unlink()
container.close()
return xczPath
def file_verification(filename,hash=False):
if not os.path.exists(cachefolder):
os.makedirs(cachefolder)
tempfolder=os.path.join(cachefolder, 'temp')
if not os.path.exists(tempfolder):
os.makedirs(tempfolder)
verdict=False;isrestored=False;cnmt_is_patched=False
if filename.endswith('.nsp') or filename.endswith('.nsx') or filename.endswith('.nsz') or filename.endswith('.xcz') or filename.endswith('.xci'):
try:
if filename.endswith('.nsp') or filename.endswith('.nsx') or filename.endswith('.nsz'):
f = squirrelNSP(filename, 'rb')
elif filename.endswith('.xci') or filename.endswith('.xcz'):
f = factory(filename)
f.open(filename, 'rb')
check,feed=f.verify()
if filename.endswith('.nsp') or filename.endswith('.nsx') or filename.endswith('.nsz'):
verdict,headerlist,feed=f.verify_sig(feed,tempfolder,cnmt='nocheck')
else:
verdict,headerlist,feed=f.verify_sig(feed,tempfolder)
output_type='nsp';multi=False;cnmtcount=0
if verdict == True:
isrestored=True
for i in range(len(headerlist)):
entry=headerlist[i]
if str(entry[0]).endswith('.cnmt.nca'):
cnmtcount+=1
if cnmt_is_patched==False:
status=entry[2]
if status=='patched':
cnmt_is_patched=True
if entry[1]!=False:
if int(entry[-1])==1:
output_type='xci'
isrestored=False
else:
pass
if isrestored == False:
if cnmt_is_patched !=True:
print('\nFILE VERIFICATION CORRECT.\n -> FILE WAS MODIFIED BUT ORIGIN IS CONFIRMED AS LEGIT\n')
else:
print('\nFILE VERIFICATION CORRECT. \n -> FILE WAS MODIFIED AND CNMT PATCHED\n')
else:
print("\nFILE VERIFICATION CORRECT. FILE IS SAFE.\n")
if verdict == False:
print("\nFILE VERIFICATION INCORRECT. \n -> UNCONFIRMED ORIGIN FILE HAS BEEN TAMPERED WITH\n")
if hash==True and verdict==True:
if filename.endswith('.nsp') or filename.endswith('.nsx') or filename.endswith('.xci'):
verdict,feed=f.verify_hash_nca(65536,headerlist,verdict,feed)
elif filename.endswith('.nsz'):
verdict,feed=f.nsz_hasher(65536,headerlist,verdict,feed)
elif filename.endswith('.xcz'):
verdict,feed=f.xcz_hasher(65536,headerlist,verdict,feed)
f.flush()
f.close()
except BaseException as e:
Print.error('Exception: ' + str(e))
return verdict,isrestored,cnmt_is_patched
def generate_and_transfer_st1(filepath,outfolder,keypatch='false'):
tname=str(os.path.basename(filepath))[:-3]+'xci'
tmpfile=os.path.join(outfolder,tname)
if filepath.endswith('xci') or filepath.endswith('xcz'):
f = factory(filepath)
f.open(filepath, 'rb')
elif filepath.endswith('nsp') or filepath.endswith('nsz'):
f = squirrelNSP(filepath, 'rb')
f.c_xci_direct(65536,tmpfile,outfolder,metapatch='true',keypatch=keypatch)
f.flush()
f.close()
def install_conv_st1(filepath, outfolder, keypatch='false'):
tname = str(os.path.basename(filepath))[:-3] + 'nsp'
tmpfile = os.path.join(outfolder, tname)
if filepath.endswith('xci'):
f = factory(filepath)
f.open(filepath, 'rb')
elif filepath.endswith('nsp'):
f = squirrelNSP(filepath, 'rb')
f.c_nsp_direct(65536, tmpfile, outfolder, keypatch=keypatch)
f.flush()
f.close()
def get_DB_dict(filepath):
installedsize=0
if filepath.endswith('xci') or filepath.endswith('xcz'):
f = factory(filepath)
f.open(filepath, 'rb')
dict=f.return_DBdict()
f.flush()
f.close()
elif filepath.endswith('nsp') or filepath.endswith('nsz') or filepath.endswith('nsx'):
f = squirrelNSP(filepath)
dict=f.return_DBdict()
f.flush()
f.close()
return dict
def __decompressNsz(filePath, outputDir, write, raiseVerificationException, statusReportInfo, pleaseNoPrint):
fileHashes = FileExistingChecks.ExtractHashes(filePath)
container = factory(filePath)
container.open(str(filePath), 'rb')
if write:
filename = changeExtension(filePath, '.nsp')
outPath = filename if outputDir == None else str(Path(outputDir).joinpath(filename))
Print.info('Decompressing %s -> %s' % (filePath, outPath), pleaseNoPrint)
with Pfs0.Pfs0Stream(outPath) as nsp:
__decompressContainer(container, nsp, fileHashes, write, raiseVerificationException, statusReportInfo, pleaseNoPrint)
else:
__decompressContainer(container, None, fileHashes, write, raiseVerificationException, statusReportInfo, pleaseNoPrint)
container.close()
def ExtractHashes(gamePath): fileHashes = set() gamePath = str(Path(gamePath).resolve()) container = factory(gamePath) container.open(gamePath, 'rb') try: for nspf in container: if isinstance(nspf, Nca.Nca) and nspf.header.contentType == Type.Content.META: for section in nspf: if isinstance(section, Pfs0.Pfs0): Cnmt = section.getCnmt() for entry in Cnmt.contentEntries: fileHashes.add(entry.hash.hex()) finally: container.close() return fileHashes
def blockCompressNsp(filePath, compressionLevel, blockSizeExponent, outputDir,
threads):
filePath = filePath.resolve()
container = factory(filePath)
container.open(str(filePath), 'rb')
nszPath = outputDir.joinpath(filePath.stem + '.nsz')
Print.info('Block compressing (level {0}) {1} -> {2}'.format(
compressionLevel, filePath, nszPath))
try:
with Pfs0.Pfs0Stream(str(nszPath)) as nsp:
blockCompressContainer(container, nsp, compressionLevel,
blockSizeExponent, threads)
except BaseException as ex:
if not ex is KeyboardInterrupt:
Print.error(format_exc())
if nszPath.is_file():
nszPath.unlink()
container.close()
return nszPath
def solidCompressNsp(filePath, compressionLevel, outputDir, threads,
stusReport, id, pleaseNoPrint):
filePath = filePath.resolve()
container = factory(filePath)
container.open(str(filePath), 'rb')
nszPath = outputDir.joinpath(filePath.stem + '.nsz')
Print.info(
'Solid compressing (level {0}) {1} -> {2}'.format(
compressionLevel, filePath, nszPath), pleaseNoPrint)
try:
with Pfs0.Pfs0Stream(str(nszPath)) as nsp:
processContainer(container, nsp, compressionLevel, threads,
stusReport, id, pleaseNoPrint)
except BaseException as ex:
if not ex is KeyboardInterrupt:
Print.error(format_exc())
if nszPath.is_file():
nszPath.unlink()
container.close()
return nszPath
def __decompress(filePath, outputDir = None, write = True, raiseVerificationException = False):
ncaHeaderSize = 0x4000
CHUNK_SZ = 0x100000
if write:
nspPath = Path(filePath[0:-1] + 'p') if outputDir == None else Path(outputDir).joinpath(Path(filePath[0:-1] + 'p').name).resolve(strict=False)
Print.info('decompressing %s -> %s' % (filePath, nspPath))
newNsp = Pfs0.Pfs0Stream(nspPath)
fileHashes = FileExistingChecks.ExtractHashes(filePath)
filePath = str(Path(filePath).resolve())
container = factory(filePath)
container.open(filePath, 'rb')
for nspf in container:
if isinstance(nspf, Nca.Nca) and nspf.header.contentType == Type.Content.DATA:
Print.info('skipping delta fragment')
continue
if not nspf._path.endswith('.ncz'):
verifyFile = nspf._path.endswith('.nca') and not nspf._path.endswith('.cnmt.nca')
if write:
f = newNsp.add(nspf._path, nspf.size)
hash = sha256()
nspf.seek(0)
while not nspf.eof():
inputChunk = nspf.read(CHUNK_SZ)
hash.update(inputChunk)
if write:
f.write(inputChunk)
if verifyFile:
if hash.hexdigest() in fileHashes:
Print.error('[VERIFIED] {0}'.format(nspf._path))
else:
Print.info('[CORRUPTED] {0}'.format(nspf._path))
if raiseVerificationException:
raise Exception("Verification detected hash missmatch!")
elif not write:
Print.info('[EXISTS] {0}'.format(nspf._path))
continue
newFileName = nspf._path[0:-1] + 'a'
if write:
f = newNsp.add(newFileName, nspf.size)
start = f.tell()
blockID = 0
nspf.seek(0)
header = nspf.read(ncaHeaderSize)
magic = nspf.read(8)
if not magic == b'NCZSECTN':
raise ValueError("No NCZSECTN found! Is this really a .ncz file?")
sectionCount = nspf.readInt64()
sections = [Header.Section(nspf) for _ in range(sectionCount)]
nca_size = ncaHeaderSize
for i in range(sectionCount):
nca_size += sections[i].size
pos = nspf.tell()
blockMagic = nspf.read(8)
nspf.seek(pos)
useBlockCompression = blockMagic == b'NCZBLOCK'
blockSize = -1
if useBlockCompression:
BlockHeader = Header.Block(nspf)
blockDecompressorReader = BlockDecompressorReader.BlockDecompressorReader(nspf, BlockHeader)
pos = nspf.tell()
if not useBlockCompression:
decompressor = ZstdDecompressor().stream_reader(nspf)
hash = sha256()
with tqdm(total=nca_size, unit_scale=True, unit="B") as bar:
if write:
f.write(header)
bar.update(len(header))
hash.update(header)
for s in sections:
i = s.offset
crypto = aes128.AESCTR(s.cryptoKey, s.cryptoCounter)
end = s.offset + s.size
while i < end:
crypto.seek(i)
chunkSz = 0x10000 if end - i > 0x10000 else end - i
if useBlockCompression:
inputChunk = blockDecompressorReader.read(chunkSz)
else:
inputChunk = decompressor.read(chunkSz)
if not len(inputChunk):
break
if not useBlockCompression:
decompressor.flush()
if s.cryptoType in (3, 4):
inputChunk = crypto.encrypt(inputChunk)
if write:
f.write(inputChunk)
hash.update(inputChunk)
i += len(inputChunk)
bar.update(chunkSz)
if hash.hexdigest() in fileHashes:
Print.error('[VERIFIED] {0}'.format(nspf._path))
else:
Print.info('[CORRUPTED] {0}'.format(nspf._path))
if raiseVerificationException:
raise Exception("Verification detected hash missmatch")
if write:
end = f.tell()
written = (end - start)
newNsp.resize(newFileName, written)
continue
if write:
newNsp.close()
container.close()
def main():
global err
try:
if len(argv) > 1:
args = ParseArguments.parse()
else:
from gui.NSZ_GUI import GUI
args = GUI().run()
if args == None:
Print.info("Done!")
return
if args.output:
outfolderToPharse = args.output
if not outfolderToPharse.endswith(
'/') and not outfolderToPharse.endswith('\\'):
outfolderToPharse += "/"
if not Path(outfolderToPharse).is_dir():
Print.error(
'Error: Output directory "{0}" does not exist!'.format(
args.output))
return
outfolder = Path(outfolderToPharse).resolve() if args.output else Path(
'.').resolve()
Print.info('')
Print.info(' NSZ v3.0 ,;:;;,')
Print.info(' ;;;;;')
Print.info(' .=\', ;:;;:,')
Print.info(' /_\', "=. \';:;:;')
Print.info(' @=:__, \,;:;:\'')
Print.info(' _(\.= ;:;;\'')
Print.info(' `"_( _/="`')
Print.info(' `"\'')
Print.info('')
barManager = enlighten.get_manager()
poolManager = Manager()
statusReport = poolManager.list()
readyForWork = Counter(0)
pleaseNoPrint = Counter(0)
pleaseKillYourself = Counter(0)
pool = []
work = poolManager.Queue()
amountOfTastkQueued = Counter(0)
if args.titlekeys:
extractTitlekeys(args.file)
if args.extract:
for f_str in args.file:
for filePath in expandFiles(Path(f_str)):
filePath_str = str(filePath)
Print.info('Extracting "{0}" to {1}'.format(
filePath_str, outfolder))
f = factory(filePath)
f.open(filePath_str, 'rb')
dir = outfolder.joinpath(filePath.stem)
f.unpack(dir, args.extractregex)
f.close()
if args.create:
Print.info('Creating "{0}"'.format(args.create))
nsp = Nsp.Nsp(None, None)
nsp.path = args.create
nsp.pack(args.file)
if args.C:
targetDictNsz = CreateTargetDict(outfolder, args.parseCnmt, ".nsz")
targetDictXcz = CreateTargetDict(outfolder, args.parseCnmt, ".xcz")
sourceFileToDelete = []
for f_str in args.file:
for filePath in expandFiles(Path(f_str)):
if not isUncompressedGame(filePath):
continue
try:
if filePath.suffix == '.nsp':
if not AllowedToWriteOutfile(
filePath, ".nsz", targetDictNsz,
args.rm_old_version, args.overwrite,
args.parseCnmt):
continue
elif filePath.suffix == '.xci':
if not AllowedToWriteOutfile(
filePath, ".xcz", targetDictXcz,
args.rm_old_version, args.overwrite,
args.parseCnmt):
continue
compress(filePath, outfolder, args, work,
amountOfTastkQueued)
if args.rm_source:
sourceFileToDelete.append(filePath)
except KeyboardInterrupt:
raise
except BaseException as e:
Print.error('Error when compressing file: %s' %
filePath)
err.append({
"filename": filePath,
"error": format_exc()
})
print_exc()
bars = []
compressedSubBars = []
BAR_FMT = u'{desc}{desc_pad}{percentage:3.0f}%|{bar}| {count:{len_total}d}/{total:d} {unit} [{elapsed}<{eta}, {rate:.2f}{unit_pad}{unit}/s]'
parallelTasks = min(args.multi, amountOfTastkQueued.value())
if parallelTasks < 0:
parallelTasks = 4
for i in range(parallelTasks):
statusReport.append([0, 0, 100])
p = Process(target=solidCompressTask,
args=(work, statusReport, readyForWork,
pleaseNoPrint, pleaseKillYourself, i))
p.start()
pool.append(p)
for i in range(parallelTasks):
bar = barManager.counter(total=100,
desc='Compressing',
unit='MiB',
color='cyan',
bar_format=BAR_FMT)
compressedSubBars.append(bar.add_subcounter('green'))
bars.append(bar)
sleep(0.02)
while readyForWork.value() < parallelTasks:
sleep(0.2)
if pleaseNoPrint.value() > 0:
continue
pleaseNoPrint.increment()
for i in range(parallelTasks):
compressedRead, compressedWritten, total = statusReport[i]
if bars[i].total != total:
bars[i].total = total // 1048576
bars[i].count = compressedRead // 1048576
compressedSubBars[i].count = compressedWritten // 1048576
bars[i].refresh()
pleaseNoPrint.decrement()
pleaseKillYourself.increment()
for i in range(readyForWork.value()):
work.put(None)
while readyForWork.value() > 0:
sleep(0.02)
for i in range(parallelTasks):
bars[i].close(clear=True)
#barManager.stop() #We aren’t using stop because of it printing garbage to the console.
for filePath in sourceFileToDelete:
delete_source_file(filePath)
if args.D:
targetDictNsz = CreateTargetDict(outfolder, args.parseCnmt, ".nsp")
targetDictXcz = CreateTargetDict(outfolder, args.parseCnmt, ".xci")
for f_str in args.file:
for filePath in expandFiles(Path(f_str)):
if not isCompressedGame(
filePath) and not isCompressedGameFile(filePath):
continue
try:
if filePath.suffix == '.nsz':
if not AllowedToWriteOutfile(
filePath, ".nsp", targetDictNsz,
args.rm_old_version, args.overwrite,
args.parseCnmt):
continue
elif filePath.suffix == '.xcz':
if not AllowedToWriteOutfile(
filePath, ".xci", targetDictXcz,
args.rm_old_version, args.overwrite,
args.parseCnmt):
continue
elif filePath.suffix == '.ncz':
outfile = changeExtension(
outfolder.joinpath(filePath.name), ".nca")
if not args.overwrite and outfile.is_file():
Print.info('{0} with the same file name already exists in the output directory.\n'\
'If you want to overwrite it use the -w parameter!'.format(outfile.name))
continue
decompress(filePath, outfolder)
if args.rm_source:
delete_source_file(filePath)
except KeyboardInterrupt:
raise
except BaseException as e:
Print.error(
'Error when decompressing file: {0}'.format(
filePath))
err.append({
"filename": filePath,
"error": format_exc()
})
print_exc()
if args.info:
for f_str in args.file:
for filePath in expandFiles(Path(f_str)):
filePath_str = str(filePath)
Print.info(filePath_str)
f = factory(filePath)
f.open(filePath_str, 'r+b')
f.printInfo(args.depth + 1)
f.close()
if args.verify and not args.C and not args.D:
for f_str in args.file:
for filePath in expandFiles(Path(f_str)):
try:
if isGame(filePath):
Print.info("[VERIFY {0}] {1}".format(
getExtensionName(filePath), filePath.name))
verify(filePath, False)
except KeyboardInterrupt:
raise
except BaseException as e:
Print.error(
'Error when verifying file: {0}'.format(filePath))
err.append({
"filename": filePath,
"error": format_exc()
})
print_exc()
if len(argv) == 1:
pass
except KeyboardInterrupt:
Print.info('Keyboard exception')
except BaseException as e:
Print.info('nut exception: {0}'.format(str(e)))
raise
if err:
Print.info(
'\n\033[93m\033[1mSummary of errors which occurred while processing files:'
)
for e in err:
Print.info('\033[0mError when processing {0}'.format(
e["filename"]))
Print.info(e["error"])
Print.info('Done!')
def solidCompress(filePath, compressionLevel=18, outputDir=None, threads=-1):
ncaHeaderSize = 0x4000
filePath = str(Path(filePath).resolve())
container = factory(filePath)
container.open(filePath, 'rb')
CHUNK_SZ = 0x1000000
nszPath = str(
Path(filePath[0:-1] + 'z' if outputDir == None else Path(outputDir).
joinpath(Path(filePath[0:-1] + 'z').name)).resolve(strict=False))
for nspf in container:
if isinstance(nspf, Ticket.Ticket):
nspf.getRightsId()
break # No need to go for other objects
Print.info('compressing (level %d) %s -> %s' %
(compressionLevel, filePath, nszPath))
newNsp = Pfs0.Pfs0Stream(nszPath)
try:
for nspf in container:
if isinstance(
nspf,
Nca.Nca) and nspf.header.contentType == Type.Content.DATA:
Print.info('skipping delta fragment')
continue
if isinstance(nspf, Nca.Nca) and (
nspf.header.contentType == Type.Content.PROGRAM
or nspf.header.contentType == Type.Content.PUBLICDATA):
if isNcaPacked(nspf, ncaHeaderSize):
newFileName = nspf._path[0:-1] + 'z'
f = newNsp.add(newFileName, nspf.size)
start = f.tell()
nspf.seek(0)
f.write(nspf.read(ncaHeaderSize))
sections = []
for fs in sortedFs(nspf):
sections += fs.getEncryptionSections()
if len(sections) == 0:
raise Exception(
"NCA can't be decrypted. Outdated keys.txt?")
header = b'NCZSECTN'
header += len(sections).to_bytes(8, 'little')
for fs in sections:
header += fs.offset.to_bytes(8, 'little')
header += fs.size.to_bytes(8, 'little')
header += fs.cryptoType.to_bytes(8, 'little')
header += b'\x00' * 8
header += fs.cryptoKey
header += fs.cryptoCounter
f.write(header)
decompressedBytes = ncaHeaderSize
with tqdm(total=nspf.size, unit_scale=True,
unit="B") as bar:
partitions = [
nspf.partition(
offset=section.offset,
size=section.size,
n=None,
cryptoType=section.cryptoType,
cryptoKey=section.cryptoKey,
cryptoCounter=bytearray(section.cryptoCounter),
autoOpen=True) for section in sections
]
partNr = 0
bar.update(f.tell())
cctx = ZstdCompressor(
level=compressionLevel, threads=threads
) if threads > 1 else ZstdCompressor(
level=compressionLevel)
compressor = cctx.stream_writer(f)
while True:
buffer = partitions[partNr].read(CHUNK_SZ)
while (len(buffer) < CHUNK_SZ
and partNr < len(partitions) - 1):
partitions[partNr].close()
partitions[partNr] = None
partNr += 1
buffer += partitions[partNr].read(CHUNK_SZ -
len(buffer))
if len(buffer) == 0:
break
compressor.write(buffer)
decompressedBytes += len(buffer)
bar.update(len(buffer))
partitions[partNr].close()
partitions[partNr] = None
compressor.flush(FLUSH_FRAME)
compressor.flush(COMPRESSOBJ_FLUSH_FINISH)
written = f.tell() - start
print('compressed %d%% %d -> %d - %s' %
(int(written * 100 / nspf.size), decompressedBytes,
written, nspf._path))
newNsp.resize(newFileName, written)
continue
else:
print('not packed!')
f = newNsp.add(nspf._path, nspf.size)
nspf.seek(0)
while not nspf.eof():
buffer = nspf.read(CHUNK_SZ)
f.write(buffer)
except KeyboardInterrupt:
remove(nszPath)
raise KeyboardInterrupt
except BaseException:
Print.error(format_exc())
remove(nszPath)
finally:
newNsp.close()
container.close()
return nszPath
def main():
global err
try:
parser = ArgumentParser()
parser.add_argument('file', nargs='*')
parser.add_argument('-C', action="store_true", help='Compress NSP')
parser.add_argument('-D', action="store_true", help='Decompress NSZ')
parser.add_argument('-l',
'--level',
type=int,
default=18,
help='Compression Level')
parser.add_argument(
'-B',
'--block',
action="store_true",
default=False,
help=
'Uses highly multithreaded block compression with random read access allowing compressed games to be played without decompression in the future however this comes with a low compression ratio cost. Current title installers do not support this yet.'
)
parser.add_argument(
'-s',
'--bs',
type=int,
default=20,
help=
'Block Size for random read access 2^x while x between 14 and 32. Default is 20 => 1 MB. Current title installers do not support this yet.'
)
parser.add_argument(
'-V',
'--verify',
action="store_true",
default=False,
help=
'Verifies files after compression raising an unhandled exception on hash mismatch and verify existing NSP and NSZ files when given as parameter'
)
parser.add_argument(
'-p',
'--parseCnmt',
action="store_true",
default=False,
help=
'Extract TitleId/Version from Cnmt if this information cannot be obtained from the filename. Required for skipping/overwriting existing files and --rm-old-version to work properly if some not every file is named properly. Supported filenames: *TitleID*[vVersion]*'
)
parser.add_argument(
'-t',
'--threads',
type=int,
default=-1,
help=
'Number of threads to compress with. Numbers < 1 corresponds to the number of logical CPU cores.'
)
parser.add_argument('-o',
'--output',
nargs='?',
help='Directory to save the output NSZ files')
parser.add_argument(
'-w',
'--overwrite',
action="store_true",
default=False,
help=
'Continues even if there already is a file with the same name or title id inside the output directory'
)
parser.add_argument('-r',
'--rm-old-version',
action="store_true",
default=False,
help='Removes older version if found')
parser.add_argument('-i',
'--info',
help='Show info about title or file')
parser.add_argument('--depth',
type=int,
default=1,
help='Max depth for file info and extraction')
parser.add_argument('-x',
'--extract',
nargs='+',
help='extract / unpack a NSP')
parser.add_argument('-c', '--create', help='create / pack a NSP')
parser.add_argument(
'--rm-source',
action='store_true',
default=False,
help=
'Deletes source file/s after compressing/decompressing. It\'s recommended to only use this in combination with --verify'
)
args = parser.parse_args()
outfolder = str(Path(args.output)) if args.output else str(Path('.'))
Print.info('')
Print.info(' NSZ v2.1 ,;:;;,')
Print.info(' ;;;;;')
Print.info(' .=\', ;:;;:,')
Print.info(' /_\', "=. \';:;:;')
Print.info(' @=:__, \,;:;:\'')
Print.info(' _(\.= ;:;;\'')
Print.info(' `"_( _/="`')
Print.info(' `"\'')
Print.info('')
if args.extract:
for filePath in args.extract:
f = factory(filePath)
f.open(filePath, 'rb')
dir = Path(Path(filePath).name).suffix[0]
f.unpack(dir)
f.close()
if args.create:
Print.info('creating ' + args.create)
nsp = Nsp.Nsp(None, None)
nsp.path = args.create
nsp.pack(args.file)
if args.C:
targetDict = CreateTargetDict(outfolder, args.parseCnmt, ".nsz")
for i in args.file:
for filePath in expandFiles(i):
try:
if filePath.endswith('.nsp'):
if not AllowedToWriteOutfile(
filePath, ".nsz", targetDict,
args.rm_old_version, args.overwrite,
args.parseCnmt):
continue
compress(filePath, args)
if args.rm_source:
delete_source_file(filePath)
except KeyboardInterrupt:
raise
except BaseException as e:
Print.error('Error when compressing file: %s' %
filePath)
err.append({
"filename": filePath,
"error": format_exc()
})
print_exc()
if args.D:
targetDict = CreateTargetDict(outfolder, args.parseCnmt, ".nsp")
for i in args.file:
for filePath in expandFiles(i):
try:
if filePath.endswith('.nsz'):
if not AllowedToWriteOutfile(
filePath, ".nsp", targetDict,
args.rm_old_version, args.overwrite,
args.parseCnmt):
continue
decompress(filePath, args.output)
if args.rm_source:
delete_source_file(filePath)
except KeyboardInterrupt:
raise
except BaseException as e:
Print.error('Error when decompressing file: %s' %
filePath)
err.append({
"filename": filePath,
"error": format_exc()
})
print_exc()
if args.info:
f = factory(args.info)
f.open(args.info, 'r+b')
f.printInfo(args.depth + 1)
if args.verify and not args.C and not args.D:
for i in args.file:
for filePath in expandFiles(i):
try:
if filePath.endswith('.nsp') or filePath.endswith(
'.nsz'):
if filePath.endswith('.nsp'):
print("[VERIFY NSP] {0}".format(i))
if filePath.endswith('.nsz'):
print("[VERIFY NSZ] {0}".format(i))
verify(filePath, False)
except KeyboardInterrupt:
raise
except BaseException as e:
Print.error('Error when verifying file: %s' % filePath)
err.append({
"filename": filePath,
"error": format_exc()
})
print_exc()
if len(argv) == 1:
pass
except KeyboardInterrupt:
Print.info('Keyboard exception')
except BaseException as e:
Print.info('nut exception: ' + str(e))
raise
if err:
Print.info('\033[93m\033[1mErrors:')
for e in err:
Print.info('\033[0mError when processing %s' % e["filename"])
Print.info(e["error"])
Print.info('Done!')
def groupncabyid_and_idoffset(filepath,
ofolder,
export,
buffer=65536,
fat='exfat',
fx='files',
nodecompress=False):
if filepath.endswith('.nsp') or filepath.endswith(
'.nsx') or filepath.endswith('.nsz'):
container = squirrelNSP(filepath, 'rb')
elif filepath.endswith('.xci') or filepath.endswith('.xcz'):
container = factory(filepath)
container.open(filepath, 'rb')
contentlist = list()
ncalist = list()
completefilelist = list()
for nspF in container.hfs0:
if str(nspF._path) == "secure":
for file in nspF:
completefilelist.append(str(file._path))
for nspF in container.hfs0:
if str(nspF._path) == "secure":
for nca in nspF:
if type(nca) == Nca:
if str(nca.header.contentType) == 'Content.META':
crypto1 = nca.header.getCryptoType()
crypto2 = nca.header.getCryptoType2()
if crypto1 == 2:
if crypto1 > crypto2:
keygen = nca.header.getCryptoType()
else:
keygen = nca.header.getCryptoType2()
else:
keygen = nca.header.getCryptoType2()
ncalist = list()
for f in nca:
for cnmt in f:
nca.rewind()
f.rewind()
cnmt.rewind()
titleid = cnmt.readInt64()
titleversion = cnmt.read(0x4)
cnmt.rewind()
cnmt.seek(0xE)
offset = cnmt.readInt16()
content_entries = cnmt.readInt16()
meta_entries = cnmt.readInt16()
content_type = str(cnmt._path)
content_type = content_type[:-22]
titleid2 = str(
hx(titleid.to_bytes(8, byteorder='big')))
titleid2 = titleid2[2:-1]
cnmt.seek(0x20)
if content_type == 'Application':
original_ID = titleid2
cnmt.readInt64()
ttag = ''
CTYPE = 'BASE'
elif content_type == 'Patch':
original_ID = cnmt.readInt64()
original_ID = str(
hx(
original_ID.to_bytes(
8, byteorder='big')))
original_ID = original_ID[2:-1]
ttag = ' [UPD]'
CTYPE = 'UPDATE'
elif content_type == 'AddOnContent':
original_ID = cnmt.readInt64()
original_ID = str(
hx(
original_ID.to_bytes(
8, byteorder='big')))
original_ID = original_ID[2:-1]
ttag = ' [DLC]'
CTYPE = 'DLC'
else:
original_ID = cnmt.readInt64()
original_ID = str(
hx(
original_ID.to_bytes(
8, byteorder='big')))
original_ID = original_ID[2:-1]
cnmt.seek(0x20 + offset)
for i in range(content_entries):
vhash = cnmt.read(0x20)
NcaId = cnmt.read(0x10)
size = cnmt.read(0x6)
ncatype = cnmt.read(0x1)
idoffset = cnmt.read(0x1)
idoffset = int.from_bytes(
idoffset,
byteorder='little',
signed=True)
#**************************************************************
version = str(
int.from_bytes(titleversion,
byteorder='little'))
version = '[v' + version + ']'
titleid3 = '[' + titleid2 + ']'
nca_name = str(hx(NcaId))
nca_name = nca_name[2:-1] + '.nca'
ncz_name = nca_name[:-1] + 'z'
if nca_name in completefilelist or ncz_name in completefilelist:
ncalist.append([nca_name, idoffset])
nca_meta = str(nca._path)
if nca_meta in completefilelist:
ncalist.append([nca_meta, 0])
target = str(nca._path)
tit_name, editor, ediver, SupLg, regionstr, isdemo = container.inf_get_title(
target, offset, content_entries,
original_ID)
tit_name = (re.sub(
r'[\/\\\:\*\?\!\"\<\>\|\.\s™©®()\~]+', ' ',
tit_name))
tit_name = tit_name.strip()
if tit_name == 'DLC' and (
str(titleid2).endswith('000')
or str(titleid2).endswith('800')):
tit_name = '-'
editor = '-'
tid = '[' + titleid2 + ']'
filename = tit_name + ' ' + tid + ' ' + version
titlerights = titleid2 + str(
'0' * 15) + str(crypto2)
contentlist.append([
filename, titleid2, titlerights, keygen,
ncalist, CTYPE, tit_name, tid, version
])
'''
for i in contentlist:
print("")
print('Filename: '+i[0])
print('TitleID: '+i[1])
print('TitleRights: '+i[2])
print('Keygen: '+str(i[3]))
for j in i[4]:
print (j)
'''
newcontentlist = []
counter = 0
for i in contentlist:
idoffsets = []
files = i[4]
metafile = None
for j in files:
if j[0].endswith('.cnmt'):
metafile = j[0]
break
for j in files:
if j[-1] not in idoffsets:
idoffsets.append(j[-1])
for j in idoffsets:
entry = copy.deepcopy(contentlist[counter])
newfiles = []
for k in files:
if k[-1] == j:
newfiles.append(k[0])
if not metafile in newfiles:
newfiles.append(metafile)
filename = entry[-3] + ' ' + (
entry[-2])[:-2] + str(j) + ' ' + version
entry[0] = filename
entry[4] = newfiles
newcontentlist.append(entry)
counter += 1
for nspF in container.hfs0:
if str(nspF._path) == "secure":
for file in nspF:
if type(file) == Ticket or file._path.endswith('.cert'):
test = file._path
test = test[0:32]
for i in newcontentlist:
if i[2] == test:
i[4].append(file._path)
elif file._path.endswith('.xml'):
test = file._path
test = test[0:-4] + '.nca'
for i in newcontentlist:
if test in i[4]:
i[4].append(file._path)
for i in newcontentlist:
if export == 'nsp':
container.cd_spl_nsp(buffer, i[0], ofolder, i[4], fat, fx,
nodecompress)
if export == 'xci':
if i[5] != 'BASE':
container.cd_spl_nsp(buffer, i[0], ofolder, i[4], fat, fx,
nodecompress)
else:
container.cd_spl_xci(buffer, i[0], ofolder, i[4], fat, fx,
nodecompress)
if export == 'both':
if i[5] != 'BASE':
container.cd_spl_nsp(buffer, i[0], ofolder, i[4], fat, fx,
nodecompress)
else:
container.cd_spl_nsp(buffer, i[0], ofolder, i[4], fat, fx,
nodecompress)
container.cd_spl_xci(buffer, i[0], ofolder, i[4], fat, fx,
nodecompress)
# def exchange_id_offset(filepath,original_idoffset,new_idoffset):
# def split_multiprogram_by_id(filepath,ofolder,output_format):
# write_cnmt_titleid
# write_cnmt_updateid
def blockCompress(filePath,
compressionLevel=18,
blockSizeExponent=20,
outputDir=None,
threads=32):
CHUNK_SZ = 0x100000
ncaHeaderSize = 0x4000
if blockSizeExponent < 14 or blockSizeExponent > 32:
raise ValueError("Block size must be between 14 and 32")
blockSize = 2**blockSizeExponent
filePath = str(Path(filePath).resolve(strict=False))
container = factory(filePath)
container.open(filePath, 'rb')
nszPath = str(
Path(filePath[0:-1] + 'z' if outputDir == None else str(
Path(outputDir).joinpath(Path(filePath[0:-1] +
'z').name))).resolve(strict=False))
Print.info('compressing (level %d) %s -> %s' %
(compressionLevel, filePath, nszPath))
newNsp = Pfs0.Pfs0Stream(nszPath)
try:
manager = Manager()
results = manager.list()
readyForWork = Counter(0)
pleaseKillYourself = Counter(0)
TasksPerChunk = 209715200 // blockSize
for i in range(TasksPerChunk):
results.append(b"")
work = manager.Queue(threads)
pool = []
for i in range(threads):
p = Process(target=compressBlockTask,
args=(work, results, readyForWork, pleaseKillYourself))
p.start()
pool.append(p)
for nspf in container:
if isinstance(
nspf,
Nca.Nca) and nspf.header.contentType == Type.Content.DATA:
Print.info('skipping delta fragment')
continue
if isinstance(nspf, Nca.Nca) and (
nspf.header.contentType == Type.Content.PROGRAM
or nspf.header.contentType == Type.Content.PUBLICDATA):
if isNcaPacked(nspf, ncaHeaderSize):
newFileName = nspf._path[0:-1] + 'z'
f = newNsp.add(newFileName, nspf.size)
start = f.tell()
nspf.seek(0)
f.write(nspf.read(ncaHeaderSize))
sections = []
for fs in sortedFs(nspf):
sections += fs.getEncryptionSections()
if len(sections) == 0:
raise Exception(
"NCA can't be decrypted. Outdated keys.txt?")
header = b'NCZSECTN'
header += len(sections).to_bytes(8, 'little')
i = 0
for fs in sections:
i += 1
header += fs.offset.to_bytes(8, 'little')
header += fs.size.to_bytes(8, 'little')
header += fs.cryptoType.to_bytes(8, 'little')
header += b'\x00' * 8
header += fs.cryptoKey
header += fs.cryptoCounter
f.write(header)
blockID = 0
chunkRelativeBlockID = 0
startChunkBlockID = 0
blocksHeaderFilePos = f.tell()
bytesToCompress = nspf.size - ncaHeaderSize
blocksToCompress = bytesToCompress // blockSize + (
bytesToCompress % blockSize > 0)
compressedblockSizeList = [0] * blocksToCompress
header = b'NCZBLOCK' #Magic
header += b'\x02' #Version
header += b'\x01' #Type
header += b'\x00' #Unused
header += blockSizeExponent.to_bytes(
1, 'little') #blockSizeExponent in bits: 2^x
header += blocksToCompress.to_bytes(
4, 'little') #Amount of Blocks
header += bytesToCompress.to_bytes(
8, 'little') #Decompressed Size
header += b'\x00' * (blocksToCompress * 4)
f.write(header)
decompressedBytes = ncaHeaderSize
with tqdm(total=nspf.size, unit_scale=True,
unit="B") as bar:
partitions = [
nspf.partition(
offset=section.offset,
size=section.size,
n=None,
cryptoType=section.cryptoType,
cryptoKey=section.cryptoKey,
cryptoCounter=bytearray(section.cryptoCounter),
autoOpen=True) for section in sections
]
partNr = 0
bar.update(f.tell())
while True:
buffer = partitions[partNr].read(blockSize)
while (len(buffer) < blockSize
and partNr < len(partitions) - 1):
partitions[partNr].close()
partitions[partNr] = None
partNr += 1
buffer += partitions[partNr].read(blockSize -
len(buffer))
if chunkRelativeBlockID >= TasksPerChunk or len(
buffer) == 0:
while readyForWork.value() < threads:
sleep(0.02)
for i in range(
min(
TasksPerChunk, blocksToCompress -
startChunkBlockID)):
compressedblockSizeList[startChunkBlockID +
i] = len(
results[i])
f.write(results[i])
results[i] = b""
if len(buffer) == 0:
pleaseKillYourself.increment()
for i in range(readyForWork.value()):
work.put(None)
while readyForWork.value() > 0:
sleep(0.02)
break
chunkRelativeBlockID = 0
startChunkBlockID = blockID
work.put([
buffer, compressionLevel,
compressedblockSizeList, chunkRelativeBlockID
])
blockID += 1
chunkRelativeBlockID += 1
decompressedBytes += len(buffer)
bar.update(len(buffer))
partitions[partNr].close()
partitions[partNr] = None
f.seek(blocksHeaderFilePos + 24)
header = b""
for compressedblockSize in compressedblockSizeList:
header += compressedblockSize.to_bytes(4, 'little')
f.write(header)
f.seek(0, 2) #Seek to end of file.
written = f.tell() - start
print('compressed %d%% %d -> %d - %s' %
(int(written * 100 / nspf.size), decompressedBytes,
written, nspf._path))
newNsp.resize(newFileName, written)
continue
else:
print('not packed!')
f = newNsp.add(nspf._path, nspf.size)
nspf.seek(0)
while not nspf.eof():
buffer = nspf.read(CHUNK_SZ)
f.write(buffer)
except KeyboardInterrupt:
remove(nszPath)
raise KeyboardInterrupt
except BaseException:
Print.error(format_exc())
remove(nszPath)
finally:
newNsp.close()
container.close()
return nszPath
def extractTitlekeys(argsFile):
titlekeysDict = {}
if Path("titlekeys.txt").is_file():
with open("titlekeys.txt", "r", encoding="utf-8") as titlekeysFile:
for line in titlekeysFile:
(rightsId, titleKey, name) = line.rstrip().split('|')
titlekeysDict[rightsId[0:16]] = (rightsId, titleKey, name)
#Print.info("Read: {0}|{1}|{2}".format(rightsId, titleKey, name))
for f_str in argsFile:
for filePath in expandFiles(Path(f_str)):
if not isNspNsz(filePath):
continue
f = factory(filePath)
f.open(str(filePath), 'rb')
ticket = f.ticket()
rightsId = format(ticket.getRightsId(), 'x').zfill(32)
titleId = rightsId[0:16]
if not titleId in titlekeysDict:
titleKey = format(ticket.getTitleKeyBlock(), 'x').zfill(32)
titlekeysDict[titleId] = (rightsId, titleKey, filePath.stem)
Print.info("Found: {0}|{1}|{2}".format(rightsId, titleKey,
filePath.stem))
else:
Print.info("Skipped already existing {0}".format(rightsId))
f.close()
Print.info("\ntitlekeys.txt:")
with open('titlekeys.txt', 'w', encoding="utf-8") as titlekeysFile:
for titleId in sorted(titlekeysDict.keys()):
(rightsId, titleKey, name) = titlekeysDict[titleId]
titleDBLine = "{0}|{1}|{2}".format(rightsId, titleKey, name)
titlekeysFile.write(titleDBLine + '\n')
Print.info(titleDBLine)
if not Path("./titledb/").is_dir():
return
Print.info("\n\ntitledb:")
Print.info("========")
titleDbPath = Path("titledb").resolve()
excludeJson = {
"cheats.json", "cnmts.json", "languages.json", "ncas.json",
"versions.json"
}
for filePath in expandFiles(titleDbPath):
if filePath.suffix == '.json':
fileName = filePath.name
if fileName in excludeJson:
continue
saveTrigger = False
Print.info("Reading {0}".format(fileName))
with open(str(filePath)) as json_file:
data = json.load(json_file)
for key, value in data.items():
titleId = value["id"]
if titleId == None:
continue
if titleId in titlekeysDict:
(rightsId, titleKey, name) = titlekeysDict[titleId]
if value["rightsId"] == None:
value["rightsId"] = rightsId
saveTrigger = True
#elif value["rightsId"] != rightsId:
# Print.info("Warn: {0} != {1}".format(value["rightsId"], rightsId))
if value["key"] == None:
Print.info("{0}: Writing key to {1}".format(
fileName, titleId))
value["key"] = titleKey
saveTrigger = True
if saveTrigger == True:
Print.info("Saving {0}".format(fileName))
with open(str(filePath), 'w') as outfile:
json.dump(data, outfile, indent=4, sort_keys=True)
Print.info("{0} saved!".format(fileName))
Print.info("")
