def blockCompressXci(filePath, compressionLevel, blockSizeExponent, outputDir, threads): filePath = filePath.resolve() container = factory(filePath) container.open(str(filePath), 'rb') secureIn = container.hfs0['secure'] xczPath = outputDir.joinpath(filePath.stem + '.xcz') Print.info('Block compressing (level {0}) {1} -> {2}'.format( compressionLevel, filePath, xczPath)) try: with Xci.XciStream( str(xczPath), originalXciPath=filePath ) as xci: # need filepath to copy XCI container settings with Hfs0.Hfs0Stream(xci.hfs0.add('secure', 0), xci.f.tell()) as secureOut: blockCompressContainer(secureIn, secureOut, compressionLevel, blockSizeExponent, threads) xci.hfs0.resize('secure', secureOut.actualSize) except BaseException as ex: if not ex is KeyboardInterrupt: Print.error(format_exc()) if xczPath.is_file(): xczPath.unlink() container.close() return xczPath
def ExtractTitleIDAndVersion(gamePath, args = None): titleId = "" version = -1 gameName = Path(gamePath).name titleIdResult = search(r'0100[0-9A-Fa-f]{12}', gameName) if args == None or not args.alwaysParseCnmt: if titleIdResult: titleId = titleIdResult.group().upper() versionResult = search(r'\[v\d+\]', gameName) if versionResult: version = int(versionResult.group()[2:-1]) if titleId != "" and version > -1 and version%65536 == 0: return(titleId, version) elif args != None and not args.parseCnmt: return None gamePath = Path(gamePath).resolve() container = factory(gamePath) container.open(str(gamePath), 'rb') if isXciXcz(gamePath): container = container.hfs0['secure'] try: for nspf in container: if isinstance(nspf, Nca.Nca) and nspf.header.contentType == Type.Content.META: for section in nspf: if isinstance(section, Pfs0.Pfs0): Cnmt = section.getCnmt() titleId = Cnmt.titleId.upper() version = Cnmt.version finally: container.close() if titleId != "" and version > -1 and version%65536 == 0: return(titleId, version) return None
def __decompressNsz(filePath, outputDir, write, raiseVerificationException, statusReportInfo, pleaseNoPrint): fileHashes = FileExistingChecks.ExtractHashes(filePath) container = factory(filePath) container.open(str(filePath), 'rb') try: if write: filename = changeExtension(filePath, '.nsp') outPath = filename if outputDir == None else str( Path(outputDir).joinpath(filename)) Print.info('Decompressing %s -> %s' % (filePath, outPath), pleaseNoPrint) with Pfs0.Pfs0Stream(outPath) as nsp: __decompressContainer(container, nsp, fileHashes, write, raiseVerificationException, statusReportInfo, pleaseNoPrint) else: __decompressContainer(container, None, fileHashes, write, raiseVerificationException, statusReportInfo, pleaseNoPrint) except BaseException: raise finally: container.close()
def __decompressXcz(filePath, outputDir, write, raiseVerificationException, statusReportInfo, pleaseNoPrint): fileHashes = FileExistingChecks.ExtractHashes(filePath) container = factory(filePath) container.open(str(filePath), 'rb') secureIn = container.hfs0['secure'] if write: filename = changeExtension(filePath, '.xci') outPath = filename if outputDir == None else str( Path(outputDir).joinpath(filename)) Print.info('Decompressing %s -> %s' % (filePath, outPath), pleaseNoPrint) with Xci.XciStream( outPath, originalXciPath=filePath ) as xci: # need filepath to copy XCI container settings with Hfs0.Hfs0Stream(xci.hfs0.add('secure', 0, pleaseNoPrint), xci.f.tell()) as secureOut: __decompressContainer(secureIn, secureOut, fileHashes, write, raiseVerificationException, statusReportInfo, pleaseNoPrint) xci.hfs0.resize('secure', secureOut.actualSize) else: __decompressContainer(secureIn, None, fileHashes, write, raiseVerificationException, statusReportInfo, pleaseNoPrint) container.close()
def decompress(filePath, outputDir, statusReportInfo, pleaseNoPrint = None): if isNspNsz(filePath): __decompressNsz(filePath, outputDir, True, False, statusReportInfo, pleaseNoPrint) elif isXciXcz(filePath): __decompressXcz(filePath, outputDir, True, False, statusReportInfo, pleaseNoPrint) elif isCompressedGameFile(filePath): filename = changeExtension(filePath, '.nca') outPath = filename if outputDir == None else str(Path(outputDir).joinpath(filename)) Print.info('Decompressing %s -> %s' % (filePath, outPath), pleaseNoPrint) try: inFile = factory(filePath) inFile.open(str(filePath), 'rb') with open(outPath, 'wb') as outFile: written, hexHash = __decompressNcz(inFile, outFile, statusReportInfo, pleaseNoPrint) fileNameHash = Path(filePath).stem.lower() if hexHash[:32] == fileNameHash: Print.info('[VERIFIED] {0}'.format(filename), pleaseNoPrint) else: Print.info('[MISMATCH] Filename startes with {0} but {1} was expected - hash verified failed!'.format(fileNameHash, hexHash[:32]), pleaseNoPrint) except BaseException as ex: if not ex is KeyboardInterrupt: Print.error(format_exc()) if Path(outPath).is_file(): Path(outPath).unlink() finally: inFile.close() else: raise NotImplementedError("Can't decompress {0} as that file format isn't implemented!".format(filePath))
def ExtractHashes(gamePath): fileHashes = set() gamePath = gamePath.resolve() container = factory(gamePath) container.open(str(gamePath), 'rb') if isXciXcz(gamePath): container = container.hfs0['secure'] try: for nspf in container: if isinstance(nspf, Nca.Nca) and nspf.header.contentType == Type.Content.META: for section in nspf: if isinstance(section, Pfs0.Pfs0): Cnmt = section.getCnmt() for entry in Cnmt.contentEntries: fileHashes.add(entry.hash.hex()) finally: container.close() return fileHashes
def blockCompressNsp(filePath, compressionLevel , blockSizeExponent, outputDir, threads): filePath = filePath.resolve() container = factory(filePath) container.open(str(filePath), 'rb') nszPath = outputDir.joinpath(filePath.stem + '.nsz') Print.info('Block compressing (level {0}) {1} -> {2}'.format(compressionLevel, filePath, nszPath)) try: with Pfs0.Pfs0Stream(str(nszPath)) as nsp: blockCompressContainer(container, nsp, compressionLevel, blockSizeExponent, threads) except BaseException as ex: if not ex is KeyboardInterrupt: Print.error(format_exc()) if nszPath.is_file(): nszPath.unlink() container.close() return nszPath
def solidCompressNsp(filePath, compressionLevel, outputDir, threads, stusReport, id, pleaseNoPrint): filePath = filePath.resolve() container = factory(filePath) container.open(str(filePath), 'rb') nszPath = outputDir.joinpath(filePath.stem + '.nsz') Print.info( 'Solid compressing (level {0}) {1} -> {2}'.format( compressionLevel, filePath, nszPath), pleaseNoPrint) try: with Pfs0.Pfs0Stream(str(nszPath)) as nsp: processContainer(container, nsp, compressionLevel, threads, stusReport, id, pleaseNoPrint) except BaseException as ex: if not ex is KeyboardInterrupt: Print.error(format_exc()) if nszPath.is_file(): nszPath.unlink() container.close() return nszPath
def main(): global err try: if len(argv) > 1: args = ParseArguments.parse() else: try: from nsz.gui.NSZ_GUI import GUI except ImportError: Print.error("Failed to import the GUI - is it installed?") return args = GUI().run() if args == None: Print.info("Done!") return if args.output: outfolderToPharse = args.output if not outfolderToPharse.endswith('/') and not outfolderToPharse.endswith('\\'): outfolderToPharse += "/" if not Path(outfolderToPharse).is_dir(): Print.error('Error: Output directory "{0}" does not exist!'.format(args.output)) return outfolder = Path(outfolderToPharse).resolve() if args.output else Path('.').resolve() Print.info('') Print.info(' NSZ v3.1 ,;:;;,') Print.info(' ;;;;;') Print.info(' .=\', ;:;;:,') Print.info(' /_\', "=. \';:;:;') Print.info(' @=:__, \,;:;:\'') Print.info(' _(\.= ;:;;\'') Print.info(' `"_( _/="`') Print.info(' `"\'') Print.info('') barManager = enlighten.get_manager() poolManager = Manager() statusReport = poolManager.list() readyForWork = Counter(0) pleaseNoPrint = Counter(0) pleaseKillYourself = Counter(0) pool = [] work = poolManager.Queue() amountOfTastkQueued = Counter(0) if args.titlekeys: extractTitlekeys(args.file) if args.extract: for f_str in args.file: for filePath in expandFiles(Path(f_str)): filePath_str = str(filePath) Print.info('Extracting "{0}" to {1}'.format(filePath_str, outfolder)) dir = outfolder.joinpath(filePath.stem) container = factory(filePath) container.open(filePath_str, 'rb') if isXciXcz(filePath): for hfs0 in container.hfs0: secureIn = hfs0 secureIn.unpack(dir.joinpath(hfs0._path), args.extractregex) else: container.unpack(dir, args.extractregex) container.close() if args.create: Print.info('Creating "{0}"'.format(args.create)) nsp = Nsp.Nsp(None, None) nsp.path = args.create nsp.pack(args.file) if args.C: targetDictNsz = CreateTargetDict(outfolder, args.parseCnmt, ".nsz") targetDictXcz = CreateTargetDict(outfolder, args.parseCnmt, ".xcz") sourceFileToDelete = [] for f_str in args.file: for filePath in expandFiles(Path(f_str)): if not isUncompressedGame(filePath): continue try: if filePath.suffix == '.nsp': if not AllowedToWriteOutfile(filePath, ".nsz", targetDictNsz, args.rm_old_version, args.overwrite, args.parseCnmt): continue elif filePath.suffix == '.xci': if not AllowedToWriteOutfile(filePath, ".xcz", targetDictXcz, args.rm_old_version, args.overwrite, args.parseCnmt): continue compress(filePath, outfolder, args, work, amountOfTastkQueued) if args.rm_source: sourceFileToDelete.append(filePath) except KeyboardInterrupt: raise except BaseException as e: Print.error('Error when compressing file: %s' % filePath) err.append({"filename":filePath,"error":format_exc() }) print_exc() bars = [] compressedSubBars = [] BAR_FMT = u'{desc}{desc_pad}{percentage:3.0f}%|{bar}| {count:{len_total}d}/{total:d} {unit} [{elapsed}<{eta}, {rate:.2f}{unit_pad}{unit}/s]' parallelTasks = min(args.multi, amountOfTastkQueued.value()) if parallelTasks < 0: parallelTasks = 4 for i in range(parallelTasks): statusReport.append([0, 0, 100]) p = Process(target=solidCompressTask, args=(work, statusReport, readyForWork, pleaseNoPrint, pleaseKillYourself, i)) p.start() pool.append(p) for i in range(parallelTasks): bar = barManager.counter(total=100, desc='Compressing', unit='MiB', color='cyan', bar_format=BAR_FMT) compressedSubBars.append(bar.add_subcounter('green')) bars.append(bar) #Ensures that all threads are started and compleaded before being requested to quit while readyForWork.value() < parallelTasks: sleep(0.2) if pleaseNoPrint.value() > 0: continue pleaseNoPrint.increment() for i in range(parallelTasks): compressedRead, compressedWritten, total = statusReport[i] if bars[i].total != total: bars[i].total = total//1048576 bars[i].count = compressedRead//1048576 compressedSubBars[i].count = compressedWritten//1048576 bars[i].refresh() pleaseNoPrint.decrement() pleaseKillYourself.increment() for i in range(readyForWork.value()): work.put(None) while readyForWork.value() > 0: sleep(0.02) for i in range(parallelTasks): bars[i].close(clear=True) #barManager.stop() #We aren’t using stop because of it printing garbage to the console. for filePath in sourceFileToDelete: delete_source_file(filePath) if args.D: targetDictNsz = CreateTargetDict(outfolder, args.parseCnmt, ".nsp") targetDictXcz = CreateTargetDict(outfolder, args.parseCnmt, ".xci") for f_str in args.file: for filePath in expandFiles(Path(f_str)): if not isCompressedGame(filePath) and not isCompressedGameFile(filePath): continue try: if filePath.suffix == '.nsz': if not AllowedToWriteOutfile(filePath, ".nsp", targetDictNsz, args.rm_old_version, args.overwrite, args.parseCnmt): continue elif filePath.suffix == '.xcz': if not AllowedToWriteOutfile(filePath, ".xci", targetDictXcz, args.rm_old_version, args.overwrite, args.parseCnmt): continue elif filePath.suffix == '.ncz': outfile = Path(changeExtension(outfolder.joinpath(filePath.name), ".nca")) if not args.overwrite and outfile.is_file(): Print.info('{0} with the same file name already exists in the output directory.\n'\ 'If you want to overwrite it use the -w parameter!'.format(outfile.name)) continue decompress(filePath, outfolder) if args.rm_source: delete_source_file(filePath) except KeyboardInterrupt: raise except BaseException as e: Print.error('Error when decompressing file: {0}'.format(filePath)) err.append({"filename":filePath, "error":format_exc()}) print_exc() if args.info: for f_str in args.file: for filePath in expandFiles(Path(f_str)): filePath_str = str(filePath) Print.info(filePath_str) f = factory(filePath) f.open(filePath_str, 'r+b') f.printInfo(args.depth+1) f.close() if args.verify and not args.C and not args.D: for f_str in args.file: for filePath in expandFiles(Path(f_str)): try: if isGame(filePath): Print.info("[VERIFY {0}] {1}".format(getExtensionName(filePath), filePath.name)) verify(filePath, False) except KeyboardInterrupt: raise except BaseException as e: Print.error('Error when verifying file: {0}'.format(filePath)) err.append({"filename":filePath,"error":format_exc()}) print_exc() if len(argv) == 1: pass except KeyboardInterrupt: Print.info('Keyboard exception') except BaseException as e: Print.info('nut exception: {0}'.format(str(e))) raise if err: Print.info('\n\033[93m\033[1mSummary of errors which occurred while processing files:') for e in err: Print.info('\033[0mError while processing {0}'.format(e["filename"])) Print.info(e["error"]) Print.info('Done!') _exit(0)
def extractTitlekeys(argsFile): titlekeysDict = {} if Path("titlekeys.txt").is_file(): with open("titlekeys.txt", "r", encoding="utf-8") as titlekeysFile: for line in titlekeysFile: (rightsId, titleKey, name) = line.rstrip().split('|') titlekeysDict[rightsId[0:16]] = (rightsId, titleKey, name) #Print.info("Read: {0}|{1}|{2}".format(rightsId, titleKey, name)) for f_str in argsFile: for filePath in expandFiles(Path(f_str)): if not isNspNsz(filePath): continue f = factory(filePath) f.open(str(filePath), 'rb') ticket = f.ticket() rightsId = format(ticket.getRightsId(), 'x').zfill(32) titleId = rightsId[0:16] if not titleId in titlekeysDict: titleKey = format(ticket.getTitleKeyBlock(), 'x').zfill(32) titlekeysDict[titleId] = (rightsId, titleKey, filePath.stem) Print.info("Found: {0}|{1}|{2}".format(rightsId, titleKey, filePath.stem)) else: Print.info("Skipped already existing {0}".format(rightsId)) f.close() Print.info("\ntitlekeys.txt:") with open('titlekeys.txt', 'w', encoding="utf-8") as titlekeysFile: for titleId in sorted(titlekeysDict.keys()): (rightsId, titleKey, name) = titlekeysDict[titleId] titleDBLine = "{0}|{1}|{2}".format(rightsId, titleKey, name) titlekeysFile.write(titleDBLine + '\n') Print.info(titleDBLine) if not Path("./titledb/").is_dir(): return Print.info("\n\ntitledb:") Print.info("========") titleDbPath = Path("titledb").resolve() excludeJson = {"cheats.json", "cnmts.json", "languages.json", "ncas.json", "versions.json"} for filePath in expandFiles(titleDbPath): if filePath.suffix == '.json': fileName = filePath.name if fileName in excludeJson: continue saveTrigger = False Print.info("Reading {0}".format(fileName)) with open(str(filePath)) as json_file: data = json.load(json_file) for key, value in data.items(): titleId = value["id"] if titleId == None: continue if titleId in titlekeysDict: (rightsId, titleKey, name) = titlekeysDict[titleId] if value["rightsId"] == None: value["rightsId"] = rightsId saveTrigger=True #elif value["rightsId"] != rightsId: # Print.info("Warn: {0} != {1}".format(value["rightsId"], rightsId)) if value["key"] == None: Print.info("{0}: Writing key to {1}".format(fileName, titleId)) value["key"] = titleKey saveTrigger=True if saveTrigger == True: Print.info("Saving {0}".format(fileName)) with open(str(filePath), 'w') as outfile: json.dump(data, outfile, indent=4, sort_keys=True) Print.info("{0} saved!".format(fileName)) Print.info("")