def __decompressNsz(filePath, outputDir, write, raiseVerificationException, statusReportInfo): fileHashes = [] # FileExistingChecks.ExtractHashes(filePath) container = factory(filePath) container.open(str(filePath), 'rb') for f in container: if isCompressedGameFile(f._path): fileHashes.append(f._path.split('.')[0]) if write: filename = changeExtension(filePath, '.nsp') outPath = filename if outputDir is None else os.path.join( outputDir, os.path.basename(filename)) Print.info('Decompressing %s -> %s' % (filePath, outPath)) if Config.dryRun: return outPath with Pfs0Stream(outPath) as nsp: __decompressContainer(container, nsp, fileHashes, write, raiseVerificationException, statusReportInfo) return outPath else: __decompressContainer(container, None, fileHashes, write, raiseVerificationException, statusReportInfo) container.close() return None
def ExtractTitleIDAndVersion(gamePath, parseCnmt): titleId = "" version = -1 gameName = Path(gamePath).name titleIdResult = search(r'0100[0-9A-Fa-f]{12}', gameName) if titleIdResult: titleId = titleIdResult.group().upper() versionResult = search(r'\[v\d+\]', gameName) if versionResult: version = int(versionResult.group()[2:-1]) if titleId != "" and version > -1 and version % 65536 == 0: return (titleId, version) elif not parseCnmt: return None gamePath = Path(gamePath).resolve() container = factory(gamePath) container.open(str(gamePath), 'rb') if isXciXcz(gamePath): container = container.hfs0['secure'] try: for nspf in container: if isinstance( nspf, Nca.Nca) and nspf.header.contentType == Type.Content.META: for section in nspf: if isinstance(section, Pfs0.Pfs0): Cnmt = section.getCnmt() titleId = Cnmt.titleId.upper() version = Cnmt.version finally: container.close() if titleId != "" and version > -1 and version % 65536 == 0: return (titleId, version) return None
def decompress(filePath, outputDir, statusReportInfo, pleaseNoPrint = None): if isNspNsz(filePath): __decompressNsz(filePath, outputDir, True, False, statusReportInfo, pleaseNoPrint) elif isXciXcz(filePath): __decompressXcz(filePath, outputDir, True, False, statusReportInfo, pleaseNoPrint) elif isCompressedGameFile(filePath): filename = changeExtension(filePath, '.nca') outPath = filename if outputDir == None else str(Path(outputDir).joinpath(filename)) Print.info('Decompressing %s -> %s' % (filePath, outPath), pleaseNoPrint) container = factory(filePath) container.open(filePath, 'rb') try: with open(outPath, 'wb') as outFile: written, hexHash = __decompressNcz(container, outFile) except BaseException as ex: if not ex is KeyboardInterrupt: Print.error(format_exc()) if outFile.is_file(): outFile.unlink() finally: container.close() fileNameHash = Path(filePath).stem.lower() if hexHash[:32] == fileNameHash: Print.info('[VERIFIED] {0}'.format(filename), pleaseNoPrint) else: Print.info('[MISMATCH] Filename startes with {0} but {1} was expected - hash verified failed!'.format(fileNameHash, hexHash[:32]), pleaseNoPrint) else: raise NotImplementedError("Can't decompress {0} as that file format isn't implemented!".format(filePath))
def __decompressXcz(filePath, outputDir, write, raiseVerificationException, statusReportInfo, pleaseNoPrint): fileHashes = FileExistingChecks.ExtractHashes(filePath) container = factory(filePath) container.open(filePath, 'rb') secureIn = container.hfs0['secure'] if write: filename = changeExtension(filePath, '.xci') outPath = filename if outputDir == None else str( Path(outputDir).joinpath(filename)) Print.info('Decompressing %s -> %s' % (filePath, outPath), pleaseNoPrint) with Xci.XciStream( outPath, originalXciPath=filePath ) as xci: # need filepath to copy XCI container settings with Hfs0.Hfs0Stream(xci.hfs0.add('secure', 0, pleaseNoPrint), xci.f.tell()) as secureOut: __decompressContainer(secureIn, secureOut, fileHashes, write, raiseVerificationException, statusReportInfo, pleaseNoPrint) xci.hfs0.resize('secure', secureOut.actualSize) else: __decompressContainer(secureIn, None, fileHashes, write, raiseVerificationException, statusReportInfo, pleaseNoPrint) container.close()
def solidCompressXci(filePath, compressionLevel, outputDir, threads, stusReport, id, pleaseNoPrint): filePath = filePath.resolve() container = factory(filePath) container.open(str(filePath), 'rb') secureIn = container.hfs0['secure'] xczPath = outputDir.joinpath(filePath.stem + '.xcz') Print.info( 'Solid compressing (level {0}) {1} -> {2}'.format( compressionLevel, filePath, xczPath), pleaseNoPrint) try: with Xci.XciStream( str(xczPath), originalXciPath=filePath ) as xci: # need filepath to copy XCI container settings with Hfs0.Hfs0Stream(xci.hfs0.add('secure', 0, pleaseNoPrint), xci.f.tell()) as secureOut: processContainer(secureIn, secureOut, compressionLevel, threads, stusReport, id, pleaseNoPrint) xci.hfs0.resize('secure', secureOut.actualSize) except BaseException as ex: if not ex is KeyboardInterrupt: Print.error(format_exc()) if xczPath.is_file(): xczPath.unlink() container.close() return xczPath
def file_verification(filename,hash=False): if not os.path.exists(cachefolder): os.makedirs(cachefolder) tempfolder=os.path.join(cachefolder, 'temp') if not os.path.exists(tempfolder): os.makedirs(tempfolder) verdict=False;isrestored=False;cnmt_is_patched=False if filename.endswith('.nsp') or filename.endswith('.nsx') or filename.endswith('.nsz') or filename.endswith('.xcz') or filename.endswith('.xci'): try: if filename.endswith('.nsp') or filename.endswith('.nsx') or filename.endswith('.nsz'): f = squirrelNSP(filename, 'rb') elif filename.endswith('.xci') or filename.endswith('.xcz'): f = factory(filename) f.open(filename, 'rb') check,feed=f.verify() if filename.endswith('.nsp') or filename.endswith('.nsx') or filename.endswith('.nsz'): verdict,headerlist,feed=f.verify_sig(feed,tempfolder,cnmt='nocheck') else: verdict,headerlist,feed=f.verify_sig(feed,tempfolder) output_type='nsp';multi=False;cnmtcount=0 if verdict == True: isrestored=True for i in range(len(headerlist)): entry=headerlist[i] if str(entry[0]).endswith('.cnmt.nca'): cnmtcount+=1 if cnmt_is_patched==False: status=entry[2] if status=='patched': cnmt_is_patched=True if entry[1]!=False: if int(entry[-1])==1: output_type='xci' isrestored=False else: pass if isrestored == False: if cnmt_is_patched !=True: print('\nFILE VERIFICATION CORRECT.\n -> FILE WAS MODIFIED BUT ORIGIN IS CONFIRMED AS LEGIT\n') else: print('\nFILE VERIFICATION CORRECT. \n -> FILE WAS MODIFIED AND CNMT PATCHED\n') else: print("\nFILE VERIFICATION CORRECT. FILE IS SAFE.\n") if verdict == False: print("\nFILE VERIFICATION INCORRECT. \n -> UNCONFIRMED ORIGIN FILE HAS BEEN TAMPERED WITH\n") if hash==True and verdict==True: if filename.endswith('.nsp') or filename.endswith('.nsx') or filename.endswith('.xci'): verdict,feed=f.verify_hash_nca(65536,headerlist,verdict,feed) elif filename.endswith('.nsz'): verdict,feed=f.nsz_hasher(65536,headerlist,verdict,feed) elif filename.endswith('.xcz'): verdict,feed=f.xcz_hasher(65536,headerlist,verdict,feed) f.flush() f.close() except BaseException as e: Print.error('Exception: ' + str(e)) return verdict,isrestored,cnmt_is_patched
def generate_and_transfer_st1(filepath,outfolder,keypatch='false'): tname=str(os.path.basename(filepath))[:-3]+'xci' tmpfile=os.path.join(outfolder,tname) if filepath.endswith('xci') or filepath.endswith('xcz'): f = factory(filepath) f.open(filepath, 'rb') elif filepath.endswith('nsp') or filepath.endswith('nsz'): f = squirrelNSP(filepath, 'rb') f.c_xci_direct(65536,tmpfile,outfolder,metapatch='true',keypatch=keypatch) f.flush() f.close()
def install_conv_st1(filepath, outfolder, keypatch='false'): tname = str(os.path.basename(filepath))[:-3] + 'nsp' tmpfile = os.path.join(outfolder, tname) if filepath.endswith('xci'): f = factory(filepath) f.open(filepath, 'rb') elif filepath.endswith('nsp'): f = squirrelNSP(filepath, 'rb') f.c_nsp_direct(65536, tmpfile, outfolder, keypatch=keypatch) f.flush() f.close()
def get_DB_dict(filepath): installedsize=0 if filepath.endswith('xci') or filepath.endswith('xcz'): f = factory(filepath) f.open(filepath, 'rb') dict=f.return_DBdict() f.flush() f.close() elif filepath.endswith('nsp') or filepath.endswith('nsz') or filepath.endswith('nsx'): f = squirrelNSP(filepath) dict=f.return_DBdict() f.flush() f.close() return dict
def __decompressNsz(filePath, outputDir, write, raiseVerificationException, statusReportInfo, pleaseNoPrint): fileHashes = FileExistingChecks.ExtractHashes(filePath) container = factory(filePath) container.open(str(filePath), 'rb') if write: filename = changeExtension(filePath, '.nsp') outPath = filename if outputDir == None else str(Path(outputDir).joinpath(filename)) Print.info('Decompressing %s -> %s' % (filePath, outPath), pleaseNoPrint) with Pfs0.Pfs0Stream(outPath) as nsp: __decompressContainer(container, nsp, fileHashes, write, raiseVerificationException, statusReportInfo, pleaseNoPrint) else: __decompressContainer(container, None, fileHashes, write, raiseVerificationException, statusReportInfo, pleaseNoPrint) container.close()
def ExtractHashes(gamePath): fileHashes = set() gamePath = str(Path(gamePath).resolve()) container = factory(gamePath) container.open(gamePath, 'rb') try: for nspf in container: if isinstance(nspf, Nca.Nca) and nspf.header.contentType == Type.Content.META: for section in nspf: if isinstance(section, Pfs0.Pfs0): Cnmt = section.getCnmt() for entry in Cnmt.contentEntries: fileHashes.add(entry.hash.hex()) finally: container.close() return fileHashes
def blockCompressNsp(filePath, compressionLevel, blockSizeExponent, outputDir, threads): filePath = filePath.resolve() container = factory(filePath) container.open(str(filePath), 'rb') nszPath = outputDir.joinpath(filePath.stem + '.nsz') Print.info('Block compressing (level {0}) {1} -> {2}'.format( compressionLevel, filePath, nszPath)) try: with Pfs0.Pfs0Stream(str(nszPath)) as nsp: blockCompressContainer(container, nsp, compressionLevel, blockSizeExponent, threads) except BaseException as ex: if not ex is KeyboardInterrupt: Print.error(format_exc()) if nszPath.is_file(): nszPath.unlink() container.close() return nszPath
def solidCompressNsp(filePath, compressionLevel, outputDir, threads, stusReport, id, pleaseNoPrint): filePath = filePath.resolve() container = factory(filePath) container.open(str(filePath), 'rb') nszPath = outputDir.joinpath(filePath.stem + '.nsz') Print.info( 'Solid compressing (level {0}) {1} -> {2}'.format( compressionLevel, filePath, nszPath), pleaseNoPrint) try: with Pfs0.Pfs0Stream(str(nszPath)) as nsp: processContainer(container, nsp, compressionLevel, threads, stusReport, id, pleaseNoPrint) except BaseException as ex: if not ex is KeyboardInterrupt: Print.error(format_exc()) if nszPath.is_file(): nszPath.unlink() container.close() return nszPath
def __decompress(filePath, outputDir = None, write = True, raiseVerificationException = False): ncaHeaderSize = 0x4000 CHUNK_SZ = 0x100000 if write: nspPath = Path(filePath[0:-1] + 'p') if outputDir == None else Path(outputDir).joinpath(Path(filePath[0:-1] + 'p').name).resolve(strict=False) Print.info('decompressing %s -> %s' % (filePath, nspPath)) newNsp = Pfs0.Pfs0Stream(nspPath) fileHashes = FileExistingChecks.ExtractHashes(filePath) filePath = str(Path(filePath).resolve()) container = factory(filePath) container.open(filePath, 'rb') for nspf in container: if isinstance(nspf, Nca.Nca) and nspf.header.contentType == Type.Content.DATA: Print.info('skipping delta fragment') continue if not nspf._path.endswith('.ncz'): verifyFile = nspf._path.endswith('.nca') and not nspf._path.endswith('.cnmt.nca') if write: f = newNsp.add(nspf._path, nspf.size) hash = sha256() nspf.seek(0) while not nspf.eof(): inputChunk = nspf.read(CHUNK_SZ) hash.update(inputChunk) if write: f.write(inputChunk) if verifyFile: if hash.hexdigest() in fileHashes: Print.error('[VERIFIED] {0}'.format(nspf._path)) else: Print.info('[CORRUPTED] {0}'.format(nspf._path)) if raiseVerificationException: raise Exception("Verification detected hash missmatch!") elif not write: Print.info('[EXISTS] {0}'.format(nspf._path)) continue newFileName = nspf._path[0:-1] + 'a' if write: f = newNsp.add(newFileName, nspf.size) start = f.tell() blockID = 0 nspf.seek(0) header = nspf.read(ncaHeaderSize) magic = nspf.read(8) if not magic == b'NCZSECTN': raise ValueError("No NCZSECTN found! Is this really a .ncz file?") sectionCount = nspf.readInt64() sections = [Header.Section(nspf) for _ in range(sectionCount)] nca_size = ncaHeaderSize for i in range(sectionCount): nca_size += sections[i].size pos = nspf.tell() blockMagic = nspf.read(8) nspf.seek(pos) useBlockCompression = blockMagic == b'NCZBLOCK' blockSize = -1 if useBlockCompression: BlockHeader = Header.Block(nspf) blockDecompressorReader = BlockDecompressorReader.BlockDecompressorReader(nspf, BlockHeader) pos = nspf.tell() if not useBlockCompression: decompressor = ZstdDecompressor().stream_reader(nspf) hash = sha256() with tqdm(total=nca_size, unit_scale=True, unit="B") as bar: if write: f.write(header) bar.update(len(header)) hash.update(header) for s in sections: i = s.offset crypto = aes128.AESCTR(s.cryptoKey, s.cryptoCounter) end = s.offset + s.size while i < end: crypto.seek(i) chunkSz = 0x10000 if end - i > 0x10000 else end - i if useBlockCompression: inputChunk = blockDecompressorReader.read(chunkSz) else: inputChunk = decompressor.read(chunkSz) if not len(inputChunk): break if not useBlockCompression: decompressor.flush() if s.cryptoType in (3, 4): inputChunk = crypto.encrypt(inputChunk) if write: f.write(inputChunk) hash.update(inputChunk) i += len(inputChunk) bar.update(chunkSz) if hash.hexdigest() in fileHashes: Print.error('[VERIFIED] {0}'.format(nspf._path)) else: Print.info('[CORRUPTED] {0}'.format(nspf._path)) if raiseVerificationException: raise Exception("Verification detected hash missmatch") if write: end = f.tell() written = (end - start) newNsp.resize(newFileName, written) continue if write: newNsp.close() container.close()
def main(): global err try: if len(argv) > 1: args = ParseArguments.parse() else: from gui.NSZ_GUI import GUI args = GUI().run() if args == None: Print.info("Done!") return if args.output: outfolderToPharse = args.output if not outfolderToPharse.endswith( '/') and not outfolderToPharse.endswith('\\'): outfolderToPharse += "/" if not Path(outfolderToPharse).is_dir(): Print.error( 'Error: Output directory "{0}" does not exist!'.format( args.output)) return outfolder = Path(outfolderToPharse).resolve() if args.output else Path( '.').resolve() Print.info('') Print.info(' NSZ v3.0 ,;:;;,') Print.info(' ;;;;;') Print.info(' .=\', ;:;;:,') Print.info(' /_\', "=. \';:;:;') Print.info(' @=:__, \,;:;:\'') Print.info(' _(\.= ;:;;\'') Print.info(' `"_( _/="`') Print.info(' `"\'') Print.info('') barManager = enlighten.get_manager() poolManager = Manager() statusReport = poolManager.list() readyForWork = Counter(0) pleaseNoPrint = Counter(0) pleaseKillYourself = Counter(0) pool = [] work = poolManager.Queue() amountOfTastkQueued = Counter(0) if args.titlekeys: extractTitlekeys(args.file) if args.extract: for f_str in args.file: for filePath in expandFiles(Path(f_str)): filePath_str = str(filePath) Print.info('Extracting "{0}" to {1}'.format( filePath_str, outfolder)) f = factory(filePath) f.open(filePath_str, 'rb') dir = outfolder.joinpath(filePath.stem) f.unpack(dir, args.extractregex) f.close() if args.create: Print.info('Creating "{0}"'.format(args.create)) nsp = Nsp.Nsp(None, None) nsp.path = args.create nsp.pack(args.file) if args.C: targetDictNsz = CreateTargetDict(outfolder, args.parseCnmt, ".nsz") targetDictXcz = CreateTargetDict(outfolder, args.parseCnmt, ".xcz") sourceFileToDelete = [] for f_str in args.file: for filePath in expandFiles(Path(f_str)): if not isUncompressedGame(filePath): continue try: if filePath.suffix == '.nsp': if not AllowedToWriteOutfile( filePath, ".nsz", targetDictNsz, args.rm_old_version, args.overwrite, args.parseCnmt): continue elif filePath.suffix == '.xci': if not AllowedToWriteOutfile( filePath, ".xcz", targetDictXcz, args.rm_old_version, args.overwrite, args.parseCnmt): continue compress(filePath, outfolder, args, work, amountOfTastkQueued) if args.rm_source: sourceFileToDelete.append(filePath) except KeyboardInterrupt: raise except BaseException as e: Print.error('Error when compressing file: %s' % filePath) err.append({ "filename": filePath, "error": format_exc() }) print_exc() bars = [] compressedSubBars = [] BAR_FMT = u'{desc}{desc_pad}{percentage:3.0f}%|{bar}| {count:{len_total}d}/{total:d} {unit} [{elapsed}<{eta}, {rate:.2f}{unit_pad}{unit}/s]' parallelTasks = min(args.multi, amountOfTastkQueued.value()) if parallelTasks < 0: parallelTasks = 4 for i in range(parallelTasks): statusReport.append([0, 0, 100]) p = Process(target=solidCompressTask, args=(work, statusReport, readyForWork, pleaseNoPrint, pleaseKillYourself, i)) p.start() pool.append(p) for i in range(parallelTasks): bar = barManager.counter(total=100, desc='Compressing', unit='MiB', color='cyan', bar_format=BAR_FMT) compressedSubBars.append(bar.add_subcounter('green')) bars.append(bar) sleep(0.02) while readyForWork.value() < parallelTasks: sleep(0.2) if pleaseNoPrint.value() > 0: continue pleaseNoPrint.increment() for i in range(parallelTasks): compressedRead, compressedWritten, total = statusReport[i] if bars[i].total != total: bars[i].total = total // 1048576 bars[i].count = compressedRead // 1048576 compressedSubBars[i].count = compressedWritten // 1048576 bars[i].refresh() pleaseNoPrint.decrement() pleaseKillYourself.increment() for i in range(readyForWork.value()): work.put(None) while readyForWork.value() > 0: sleep(0.02) for i in range(parallelTasks): bars[i].close(clear=True) #barManager.stop() #We aren’t using stop because of it printing garbage to the console. for filePath in sourceFileToDelete: delete_source_file(filePath) if args.D: targetDictNsz = CreateTargetDict(outfolder, args.parseCnmt, ".nsp") targetDictXcz = CreateTargetDict(outfolder, args.parseCnmt, ".xci") for f_str in args.file: for filePath in expandFiles(Path(f_str)): if not isCompressedGame( filePath) and not isCompressedGameFile(filePath): continue try: if filePath.suffix == '.nsz': if not AllowedToWriteOutfile( filePath, ".nsp", targetDictNsz, args.rm_old_version, args.overwrite, args.parseCnmt): continue elif filePath.suffix == '.xcz': if not AllowedToWriteOutfile( filePath, ".xci", targetDictXcz, args.rm_old_version, args.overwrite, args.parseCnmt): continue elif filePath.suffix == '.ncz': outfile = changeExtension( outfolder.joinpath(filePath.name), ".nca") if not args.overwrite and outfile.is_file(): Print.info('{0} with the same file name already exists in the output directory.\n'\ 'If you want to overwrite it use the -w parameter!'.format(outfile.name)) continue decompress(filePath, outfolder) if args.rm_source: delete_source_file(filePath) except KeyboardInterrupt: raise except BaseException as e: Print.error( 'Error when decompressing file: {0}'.format( filePath)) err.append({ "filename": filePath, "error": format_exc() }) print_exc() if args.info: for f_str in args.file: for filePath in expandFiles(Path(f_str)): filePath_str = str(filePath) Print.info(filePath_str) f = factory(filePath) f.open(filePath_str, 'r+b') f.printInfo(args.depth + 1) f.close() if args.verify and not args.C and not args.D: for f_str in args.file: for filePath in expandFiles(Path(f_str)): try: if isGame(filePath): Print.info("[VERIFY {0}] {1}".format( getExtensionName(filePath), filePath.name)) verify(filePath, False) except KeyboardInterrupt: raise except BaseException as e: Print.error( 'Error when verifying file: {0}'.format(filePath)) err.append({ "filename": filePath, "error": format_exc() }) print_exc() if len(argv) == 1: pass except KeyboardInterrupt: Print.info('Keyboard exception') except BaseException as e: Print.info('nut exception: {0}'.format(str(e))) raise if err: Print.info( '\n\033[93m\033[1mSummary of errors which occurred while processing files:' ) for e in err: Print.info('\033[0mError when processing {0}'.format( e["filename"])) Print.info(e["error"]) Print.info('Done!')
def solidCompress(filePath, compressionLevel=18, outputDir=None, threads=-1): ncaHeaderSize = 0x4000 filePath = str(Path(filePath).resolve()) container = factory(filePath) container.open(filePath, 'rb') CHUNK_SZ = 0x1000000 nszPath = str( Path(filePath[0:-1] + 'z' if outputDir == None else Path(outputDir). joinpath(Path(filePath[0:-1] + 'z').name)).resolve(strict=False)) for nspf in container: if isinstance(nspf, Ticket.Ticket): nspf.getRightsId() break # No need to go for other objects Print.info('compressing (level %d) %s -> %s' % (compressionLevel, filePath, nszPath)) newNsp = Pfs0.Pfs0Stream(nszPath) try: for nspf in container: if isinstance( nspf, Nca.Nca) and nspf.header.contentType == Type.Content.DATA: Print.info('skipping delta fragment') continue if isinstance(nspf, Nca.Nca) and ( nspf.header.contentType == Type.Content.PROGRAM or nspf.header.contentType == Type.Content.PUBLICDATA): if isNcaPacked(nspf, ncaHeaderSize): newFileName = nspf._path[0:-1] + 'z' f = newNsp.add(newFileName, nspf.size) start = f.tell() nspf.seek(0) f.write(nspf.read(ncaHeaderSize)) sections = [] for fs in sortedFs(nspf): sections += fs.getEncryptionSections() if len(sections) == 0: raise Exception( "NCA can't be decrypted. Outdated keys.txt?") header = b'NCZSECTN' header += len(sections).to_bytes(8, 'little') for fs in sections: header += fs.offset.to_bytes(8, 'little') header += fs.size.to_bytes(8, 'little') header += fs.cryptoType.to_bytes(8, 'little') header += b'\x00' * 8 header += fs.cryptoKey header += fs.cryptoCounter f.write(header) decompressedBytes = ncaHeaderSize with tqdm(total=nspf.size, unit_scale=True, unit="B") as bar: partitions = [ nspf.partition( offset=section.offset, size=section.size, n=None, cryptoType=section.cryptoType, cryptoKey=section.cryptoKey, cryptoCounter=bytearray(section.cryptoCounter), autoOpen=True) for section in sections ] partNr = 0 bar.update(f.tell()) cctx = ZstdCompressor( level=compressionLevel, threads=threads ) if threads > 1 else ZstdCompressor( level=compressionLevel) compressor = cctx.stream_writer(f) while True: buffer = partitions[partNr].read(CHUNK_SZ) while (len(buffer) < CHUNK_SZ and partNr < len(partitions) - 1): partitions[partNr].close() partitions[partNr] = None partNr += 1 buffer += partitions[partNr].read(CHUNK_SZ - len(buffer)) if len(buffer) == 0: break compressor.write(buffer) decompressedBytes += len(buffer) bar.update(len(buffer)) partitions[partNr].close() partitions[partNr] = None compressor.flush(FLUSH_FRAME) compressor.flush(COMPRESSOBJ_FLUSH_FINISH) written = f.tell() - start print('compressed %d%% %d -> %d - %s' % (int(written * 100 / nspf.size), decompressedBytes, written, nspf._path)) newNsp.resize(newFileName, written) continue else: print('not packed!') f = newNsp.add(nspf._path, nspf.size) nspf.seek(0) while not nspf.eof(): buffer = nspf.read(CHUNK_SZ) f.write(buffer) except KeyboardInterrupt: remove(nszPath) raise KeyboardInterrupt except BaseException: Print.error(format_exc()) remove(nszPath) finally: newNsp.close() container.close() return nszPath
def main(): global err try: parser = ArgumentParser() parser.add_argument('file', nargs='*') parser.add_argument('-C', action="store_true", help='Compress NSP') parser.add_argument('-D', action="store_true", help='Decompress NSZ') parser.add_argument('-l', '--level', type=int, default=18, help='Compression Level') parser.add_argument( '-B', '--block', action="store_true", default=False, help= 'Uses highly multithreaded block compression with random read access allowing compressed games to be played without decompression in the future however this comes with a low compression ratio cost. Current title installers do not support this yet.' ) parser.add_argument( '-s', '--bs', type=int, default=20, help= 'Block Size for random read access 2^x while x between 14 and 32. Default is 20 => 1 MB. Current title installers do not support this yet.' ) parser.add_argument( '-V', '--verify', action="store_true", default=False, help= 'Verifies files after compression raising an unhandled exception on hash mismatch and verify existing NSP and NSZ files when given as parameter' ) parser.add_argument( '-p', '--parseCnmt', action="store_true", default=False, help= 'Extract TitleId/Version from Cnmt if this information cannot be obtained from the filename. Required for skipping/overwriting existing files and --rm-old-version to work properly if some not every file is named properly. Supported filenames: *TitleID*[vVersion]*' ) parser.add_argument( '-t', '--threads', type=int, default=-1, help= 'Number of threads to compress with. Numbers < 1 corresponds to the number of logical CPU cores.' ) parser.add_argument('-o', '--output', nargs='?', help='Directory to save the output NSZ files') parser.add_argument( '-w', '--overwrite', action="store_true", default=False, help= 'Continues even if there already is a file with the same name or title id inside the output directory' ) parser.add_argument('-r', '--rm-old-version', action="store_true", default=False, help='Removes older version if found') parser.add_argument('-i', '--info', help='Show info about title or file') parser.add_argument('--depth', type=int, default=1, help='Max depth for file info and extraction') parser.add_argument('-x', '--extract', nargs='+', help='extract / unpack a NSP') parser.add_argument('-c', '--create', help='create / pack a NSP') parser.add_argument( '--rm-source', action='store_true', default=False, help= 'Deletes source file/s after compressing/decompressing. It\'s recommended to only use this in combination with --verify' ) args = parser.parse_args() outfolder = str(Path(args.output)) if args.output else str(Path('.')) Print.info('') Print.info(' NSZ v2.1 ,;:;;,') Print.info(' ;;;;;') Print.info(' .=\', ;:;;:,') Print.info(' /_\', "=. \';:;:;') Print.info(' @=:__, \,;:;:\'') Print.info(' _(\.= ;:;;\'') Print.info(' `"_( _/="`') Print.info(' `"\'') Print.info('') if args.extract: for filePath in args.extract: f = factory(filePath) f.open(filePath, 'rb') dir = Path(Path(filePath).name).suffix[0] f.unpack(dir) f.close() if args.create: Print.info('creating ' + args.create) nsp = Nsp.Nsp(None, None) nsp.path = args.create nsp.pack(args.file) if args.C: targetDict = CreateTargetDict(outfolder, args.parseCnmt, ".nsz") for i in args.file: for filePath in expandFiles(i): try: if filePath.endswith('.nsp'): if not AllowedToWriteOutfile( filePath, ".nsz", targetDict, args.rm_old_version, args.overwrite, args.parseCnmt): continue compress(filePath, args) if args.rm_source: delete_source_file(filePath) except KeyboardInterrupt: raise except BaseException as e: Print.error('Error when compressing file: %s' % filePath) err.append({ "filename": filePath, "error": format_exc() }) print_exc() if args.D: targetDict = CreateTargetDict(outfolder, args.parseCnmt, ".nsp") for i in args.file: for filePath in expandFiles(i): try: if filePath.endswith('.nsz'): if not AllowedToWriteOutfile( filePath, ".nsp", targetDict, args.rm_old_version, args.overwrite, args.parseCnmt): continue decompress(filePath, args.output) if args.rm_source: delete_source_file(filePath) except KeyboardInterrupt: raise except BaseException as e: Print.error('Error when decompressing file: %s' % filePath) err.append({ "filename": filePath, "error": format_exc() }) print_exc() if args.info: f = factory(args.info) f.open(args.info, 'r+b') f.printInfo(args.depth + 1) if args.verify and not args.C and not args.D: for i in args.file: for filePath in expandFiles(i): try: if filePath.endswith('.nsp') or filePath.endswith( '.nsz'): if filePath.endswith('.nsp'): print("[VERIFY NSP] {0}".format(i)) if filePath.endswith('.nsz'): print("[VERIFY NSZ] {0}".format(i)) verify(filePath, False) except KeyboardInterrupt: raise except BaseException as e: Print.error('Error when verifying file: %s' % filePath) err.append({ "filename": filePath, "error": format_exc() }) print_exc() if len(argv) == 1: pass except KeyboardInterrupt: Print.info('Keyboard exception') except BaseException as e: Print.info('nut exception: ' + str(e)) raise if err: Print.info('\033[93m\033[1mErrors:') for e in err: Print.info('\033[0mError when processing %s' % e["filename"]) Print.info(e["error"]) Print.info('Done!')
def groupncabyid_and_idoffset(filepath, ofolder, export, buffer=65536, fat='exfat', fx='files', nodecompress=False): if filepath.endswith('.nsp') or filepath.endswith( '.nsx') or filepath.endswith('.nsz'): container = squirrelNSP(filepath, 'rb') elif filepath.endswith('.xci') or filepath.endswith('.xcz'): container = factory(filepath) container.open(filepath, 'rb') contentlist = list() ncalist = list() completefilelist = list() for nspF in container.hfs0: if str(nspF._path) == "secure": for file in nspF: completefilelist.append(str(file._path)) for nspF in container.hfs0: if str(nspF._path) == "secure": for nca in nspF: if type(nca) == Nca: if str(nca.header.contentType) == 'Content.META': crypto1 = nca.header.getCryptoType() crypto2 = nca.header.getCryptoType2() if crypto1 == 2: if crypto1 > crypto2: keygen = nca.header.getCryptoType() else: keygen = nca.header.getCryptoType2() else: keygen = nca.header.getCryptoType2() ncalist = list() for f in nca: for cnmt in f: nca.rewind() f.rewind() cnmt.rewind() titleid = cnmt.readInt64() titleversion = cnmt.read(0x4) cnmt.rewind() cnmt.seek(0xE) offset = cnmt.readInt16() content_entries = cnmt.readInt16() meta_entries = cnmt.readInt16() content_type = str(cnmt._path) content_type = content_type[:-22] titleid2 = str( hx(titleid.to_bytes(8, byteorder='big'))) titleid2 = titleid2[2:-1] cnmt.seek(0x20) if content_type == 'Application': original_ID = titleid2 cnmt.readInt64() ttag = '' CTYPE = 'BASE' elif content_type == 'Patch': original_ID = cnmt.readInt64() original_ID = str( hx( original_ID.to_bytes( 8, byteorder='big'))) original_ID = original_ID[2:-1] ttag = ' [UPD]' CTYPE = 'UPDATE' elif content_type == 'AddOnContent': original_ID = cnmt.readInt64() original_ID = str( hx( original_ID.to_bytes( 8, byteorder='big'))) original_ID = original_ID[2:-1] ttag = ' [DLC]' CTYPE = 'DLC' else: original_ID = cnmt.readInt64() original_ID = str( hx( original_ID.to_bytes( 8, byteorder='big'))) original_ID = original_ID[2:-1] cnmt.seek(0x20 + offset) for i in range(content_entries): vhash = cnmt.read(0x20) NcaId = cnmt.read(0x10) size = cnmt.read(0x6) ncatype = cnmt.read(0x1) idoffset = cnmt.read(0x1) idoffset = int.from_bytes( idoffset, byteorder='little', signed=True) #************************************************************** version = str( int.from_bytes(titleversion, byteorder='little')) version = '[v' + version + ']' titleid3 = '[' + titleid2 + ']' nca_name = str(hx(NcaId)) nca_name = nca_name[2:-1] + '.nca' ncz_name = nca_name[:-1] + 'z' if nca_name in completefilelist or ncz_name in completefilelist: ncalist.append([nca_name, idoffset]) nca_meta = str(nca._path) if nca_meta in completefilelist: ncalist.append([nca_meta, 0]) target = str(nca._path) tit_name, editor, ediver, SupLg, regionstr, isdemo = container.inf_get_title( target, offset, content_entries, original_ID) tit_name = (re.sub( r'[\/\\\:\*\?\!\"\<\>\|\.\s™©®()\~]+', ' ', tit_name)) tit_name = tit_name.strip() if tit_name == 'DLC' and ( str(titleid2).endswith('000') or str(titleid2).endswith('800')): tit_name = '-' editor = '-' tid = '[' + titleid2 + ']' filename = tit_name + ' ' + tid + ' ' + version titlerights = titleid2 + str( '0' * 15) + str(crypto2) contentlist.append([ filename, titleid2, titlerights, keygen, ncalist, CTYPE, tit_name, tid, version ]) ''' for i in contentlist: print("") print('Filename: '+i[0]) print('TitleID: '+i[1]) print('TitleRights: '+i[2]) print('Keygen: '+str(i[3])) for j in i[4]: print (j) ''' newcontentlist = [] counter = 0 for i in contentlist: idoffsets = [] files = i[4] metafile = None for j in files: if j[0].endswith('.cnmt'): metafile = j[0] break for j in files: if j[-1] not in idoffsets: idoffsets.append(j[-1]) for j in idoffsets: entry = copy.deepcopy(contentlist[counter]) newfiles = [] for k in files: if k[-1] == j: newfiles.append(k[0]) if not metafile in newfiles: newfiles.append(metafile) filename = entry[-3] + ' ' + ( entry[-2])[:-2] + str(j) + ' ' + version entry[0] = filename entry[4] = newfiles newcontentlist.append(entry) counter += 1 for nspF in container.hfs0: if str(nspF._path) == "secure": for file in nspF: if type(file) == Ticket or file._path.endswith('.cert'): test = file._path test = test[0:32] for i in newcontentlist: if i[2] == test: i[4].append(file._path) elif file._path.endswith('.xml'): test = file._path test = test[0:-4] + '.nca' for i in newcontentlist: if test in i[4]: i[4].append(file._path) for i in newcontentlist: if export == 'nsp': container.cd_spl_nsp(buffer, i[0], ofolder, i[4], fat, fx, nodecompress) if export == 'xci': if i[5] != 'BASE': container.cd_spl_nsp(buffer, i[0], ofolder, i[4], fat, fx, nodecompress) else: container.cd_spl_xci(buffer, i[0], ofolder, i[4], fat, fx, nodecompress) if export == 'both': if i[5] != 'BASE': container.cd_spl_nsp(buffer, i[0], ofolder, i[4], fat, fx, nodecompress) else: container.cd_spl_nsp(buffer, i[0], ofolder, i[4], fat, fx, nodecompress) container.cd_spl_xci(buffer, i[0], ofolder, i[4], fat, fx, nodecompress) # def exchange_id_offset(filepath,original_idoffset,new_idoffset): # def split_multiprogram_by_id(filepath,ofolder,output_format): # write_cnmt_titleid # write_cnmt_updateid
def blockCompress(filePath, compressionLevel=18, blockSizeExponent=20, outputDir=None, threads=32): CHUNK_SZ = 0x100000 ncaHeaderSize = 0x4000 if blockSizeExponent < 14 or blockSizeExponent > 32: raise ValueError("Block size must be between 14 and 32") blockSize = 2**blockSizeExponent filePath = str(Path(filePath).resolve(strict=False)) container = factory(filePath) container.open(filePath, 'rb') nszPath = str( Path(filePath[0:-1] + 'z' if outputDir == None else str( Path(outputDir).joinpath(Path(filePath[0:-1] + 'z').name))).resolve(strict=False)) Print.info('compressing (level %d) %s -> %s' % (compressionLevel, filePath, nszPath)) newNsp = Pfs0.Pfs0Stream(nszPath) try: manager = Manager() results = manager.list() readyForWork = Counter(0) pleaseKillYourself = Counter(0) TasksPerChunk = 209715200 // blockSize for i in range(TasksPerChunk): results.append(b"") work = manager.Queue(threads) pool = [] for i in range(threads): p = Process(target=compressBlockTask, args=(work, results, readyForWork, pleaseKillYourself)) p.start() pool.append(p) for nspf in container: if isinstance( nspf, Nca.Nca) and nspf.header.contentType == Type.Content.DATA: Print.info('skipping delta fragment') continue if isinstance(nspf, Nca.Nca) and ( nspf.header.contentType == Type.Content.PROGRAM or nspf.header.contentType == Type.Content.PUBLICDATA): if isNcaPacked(nspf, ncaHeaderSize): newFileName = nspf._path[0:-1] + 'z' f = newNsp.add(newFileName, nspf.size) start = f.tell() nspf.seek(0) f.write(nspf.read(ncaHeaderSize)) sections = [] for fs in sortedFs(nspf): sections += fs.getEncryptionSections() if len(sections) == 0: raise Exception( "NCA can't be decrypted. Outdated keys.txt?") header = b'NCZSECTN' header += len(sections).to_bytes(8, 'little') i = 0 for fs in sections: i += 1 header += fs.offset.to_bytes(8, 'little') header += fs.size.to_bytes(8, 'little') header += fs.cryptoType.to_bytes(8, 'little') header += b'\x00' * 8 header += fs.cryptoKey header += fs.cryptoCounter f.write(header) blockID = 0 chunkRelativeBlockID = 0 startChunkBlockID = 0 blocksHeaderFilePos = f.tell() bytesToCompress = nspf.size - ncaHeaderSize blocksToCompress = bytesToCompress // blockSize + ( bytesToCompress % blockSize > 0) compressedblockSizeList = [0] * blocksToCompress header = b'NCZBLOCK' #Magic header += b'\x02' #Version header += b'\x01' #Type header += b'\x00' #Unused header += blockSizeExponent.to_bytes( 1, 'little') #blockSizeExponent in bits: 2^x header += blocksToCompress.to_bytes( 4, 'little') #Amount of Blocks header += bytesToCompress.to_bytes( 8, 'little') #Decompressed Size header += b'\x00' * (blocksToCompress * 4) f.write(header) decompressedBytes = ncaHeaderSize with tqdm(total=nspf.size, unit_scale=True, unit="B") as bar: partitions = [ nspf.partition( offset=section.offset, size=section.size, n=None, cryptoType=section.cryptoType, cryptoKey=section.cryptoKey, cryptoCounter=bytearray(section.cryptoCounter), autoOpen=True) for section in sections ] partNr = 0 bar.update(f.tell()) while True: buffer = partitions[partNr].read(blockSize) while (len(buffer) < blockSize and partNr < len(partitions) - 1): partitions[partNr].close() partitions[partNr] = None partNr += 1 buffer += partitions[partNr].read(blockSize - len(buffer)) if chunkRelativeBlockID >= TasksPerChunk or len( buffer) == 0: while readyForWork.value() < threads: sleep(0.02) for i in range( min( TasksPerChunk, blocksToCompress - startChunkBlockID)): compressedblockSizeList[startChunkBlockID + i] = len( results[i]) f.write(results[i]) results[i] = b"" if len(buffer) == 0: pleaseKillYourself.increment() for i in range(readyForWork.value()): work.put(None) while readyForWork.value() > 0: sleep(0.02) break chunkRelativeBlockID = 0 startChunkBlockID = blockID work.put([ buffer, compressionLevel, compressedblockSizeList, chunkRelativeBlockID ]) blockID += 1 chunkRelativeBlockID += 1 decompressedBytes += len(buffer) bar.update(len(buffer)) partitions[partNr].close() partitions[partNr] = None f.seek(blocksHeaderFilePos + 24) header = b"" for compressedblockSize in compressedblockSizeList: header += compressedblockSize.to_bytes(4, 'little') f.write(header) f.seek(0, 2) #Seek to end of file. written = f.tell() - start print('compressed %d%% %d -> %d - %s' % (int(written * 100 / nspf.size), decompressedBytes, written, nspf._path)) newNsp.resize(newFileName, written) continue else: print('not packed!') f = newNsp.add(nspf._path, nspf.size) nspf.seek(0) while not nspf.eof(): buffer = nspf.read(CHUNK_SZ) f.write(buffer) except KeyboardInterrupt: remove(nszPath) raise KeyboardInterrupt except BaseException: Print.error(format_exc()) remove(nszPath) finally: newNsp.close() container.close() return nszPath
def extractTitlekeys(argsFile): titlekeysDict = {} if Path("titlekeys.txt").is_file(): with open("titlekeys.txt", "r", encoding="utf-8") as titlekeysFile: for line in titlekeysFile: (rightsId, titleKey, name) = line.rstrip().split('|') titlekeysDict[rightsId[0:16]] = (rightsId, titleKey, name) #Print.info("Read: {0}|{1}|{2}".format(rightsId, titleKey, name)) for f_str in argsFile: for filePath in expandFiles(Path(f_str)): if not isNspNsz(filePath): continue f = factory(filePath) f.open(str(filePath), 'rb') ticket = f.ticket() rightsId = format(ticket.getRightsId(), 'x').zfill(32) titleId = rightsId[0:16] if not titleId in titlekeysDict: titleKey = format(ticket.getTitleKeyBlock(), 'x').zfill(32) titlekeysDict[titleId] = (rightsId, titleKey, filePath.stem) Print.info("Found: {0}|{1}|{2}".format(rightsId, titleKey, filePath.stem)) else: Print.info("Skipped already existing {0}".format(rightsId)) f.close() Print.info("\ntitlekeys.txt:") with open('titlekeys.txt', 'w', encoding="utf-8") as titlekeysFile: for titleId in sorted(titlekeysDict.keys()): (rightsId, titleKey, name) = titlekeysDict[titleId] titleDBLine = "{0}|{1}|{2}".format(rightsId, titleKey, name) titlekeysFile.write(titleDBLine + '\n') Print.info(titleDBLine) if not Path("./titledb/").is_dir(): return Print.info("\n\ntitledb:") Print.info("========") titleDbPath = Path("titledb").resolve() excludeJson = { "cheats.json", "cnmts.json", "languages.json", "ncas.json", "versions.json" } for filePath in expandFiles(titleDbPath): if filePath.suffix == '.json': fileName = filePath.name if fileName in excludeJson: continue saveTrigger = False Print.info("Reading {0}".format(fileName)) with open(str(filePath)) as json_file: data = json.load(json_file) for key, value in data.items(): titleId = value["id"] if titleId == None: continue if titleId in titlekeysDict: (rightsId, titleKey, name) = titlekeysDict[titleId] if value["rightsId"] == None: value["rightsId"] = rightsId saveTrigger = True #elif value["rightsId"] != rightsId: # Print.info("Warn: {0} != {1}".format(value["rightsId"], rightsId)) if value["key"] == None: Print.info("{0}: Writing key to {1}".format( fileName, titleId)) value["key"] = titleKey saveTrigger = True if saveTrigger == True: Print.info("Saving {0}".format(fileName)) with open(str(filePath), 'w') as outfile: json.dump(data, outfile, indent=4, sort_keys=True) Print.info("{0} saved!".format(fileName)) Print.info("")