def __decompressNcz(nspf, f): ncaHeaderSize = 0x4000 blockID = 0 nspf.seek(0) header = nspf.read(ncaHeaderSize) start = f.tell() magic = nspf.read(8) if not magic == b'NCZSECTN': raise ValueError("No NCZSECTN found! Is this really a .ncz file?") sectionCount = readInt64(nspf) sections = [Section(nspf) for _ in range(sectionCount)] nca_size = ncaHeaderSize for i in range(sectionCount): nca_size += sections[i].size pos = nspf.tell() blockMagic = nspf.read(8) nspf.seek(pos) useBlockCompression = blockMagic == b'NCZBLOCK' blockSize = -1 if useBlockCompression: BlockHeader = Block(nspf) blockDecompressorReader = BlockDecompressorReader.BlockDecompressorReader( nspf, BlockHeader) pos = nspf.tell() if not useBlockCompression: decompressor = ZstdDecompressor().stream_reader(nspf) hash = sha256() f.write(header) hash.update(header) for s in sections: i = s.offset crypto = AESCTR(s.cryptoKey, s.cryptoCounter) end = s.offset + s.size while i < end: crypto.seek(i) chunkSz = 0x10000 if end - i > 0x10000 else end - i if useBlockCompression: inputChunk = blockDecompressorReader.read(chunkSz) else: inputChunk = decompressor.read(chunkSz) if not len(inputChunk): break if not useBlockCompression: decompressor.flush() if s.cryptoType in (3, 4): inputChunk = crypto.encrypt(inputChunk) f.write(inputChunk) hash.update(inputChunk) i += len(inputChunk) hexHash = hash.hexdigest() end = f.tell() written = (end - start) return (written, hexHash)
def __decompressNcz(nspf, f, statusReportInfo, pleaseNoPrint): ncaHeaderSize = 0x4000 blockID = 0 nspf.seek(0) header = nspf.read(ncaHeaderSize) if f != None: start = f.tell() magic = nspf.read(8) if not magic == b'NCZSECTN': raise ValueError("No NCZSECTN found! Is this really a .ncz file?") sectionCount = nspf.readInt64() sections = [Header.Section(nspf) for _ in range(sectionCount)] nca_size = ncaHeaderSize for i in range(sectionCount): nca_size += sections[i].size pos = nspf.tell() blockMagic = nspf.read(8) nspf.seek(pos) useBlockCompression = blockMagic == b'NCZBLOCK' blockSize = -1 if useBlockCompression: BlockHeader = Header.Block(nspf) blockDecompressorReader = BlockDecompressorReader.BlockDecompressorReader( nspf, BlockHeader) pos = nspf.tell() if not useBlockCompression: decompressor = ZstdDecompressor().stream_reader(nspf) hash = sha256() if statusReportInfo == None: BAR_FMT = u'{desc}{desc_pad}{percentage:3.0f}%|{bar}| {count:{len_total}d}/{total:d} {unit} [{elapsed}<{eta}, {rate:.2f}{unit_pad}{unit}/s]' bar = enlighten.Counter(total=nca_size // 1048576, desc='Decompress', unit="MiB", color='red', bar_format=BAR_FMT) decompressedBytes = len(header) if f != None: f.write(header) if statusReportInfo != None: statusReport, id = statusReportInfo statusReport[id] = [len(header), 0, nca_size] else: bar.count = decompressedBytes // 1048576 bar.refresh() hash.update(header) for s in sections: i = s.offset crypto = aes128.AESCTR(s.cryptoKey, s.cryptoCounter) end = s.offset + s.size while i < end: crypto.seek(i) chunkSz = 0x10000 if end - i > 0x10000 else end - i if useBlockCompression: inputChunk = blockDecompressorReader.read(chunkSz) else: inputChunk = decompressor.read(chunkSz) if not len(inputChunk): break if not useBlockCompression: decompressor.flush() if s.cryptoType in (3, 4): inputChunk = crypto.encrypt(inputChunk) if f != None: f.write(inputChunk) hash.update(inputChunk) lenInputChunk = len(inputChunk) i += lenInputChunk decompressedBytes += lenInputChunk if statusReportInfo != None: statusReport[id] = [ statusReport[id][0] + chunkSz, statusReport[id][1], nca_size ] else: bar.count = decompressedBytes // 1048576 bar.refresh() bar.close() hexHash = hash.hexdigest() if f != None: end = f.tell() written = (end - start) return (written, hexHash) return (0, hexHash)
def __decompressNcz(nspf, f, statusReportInfo, pleaseNoPrint): UNCOMPRESSABLE_HEADER_SIZE = 0x4000 blockID = 0 nspf.seek(0) header = nspf.read(UNCOMPRESSABLE_HEADER_SIZE) if f != None: start = f.tell() magic = nspf.read(8) if not magic == b'NCZSECTN': raise ValueError("No NCZSECTN found! Is this really a .ncz file?") sectionCount = nspf.readInt64() sections = [Header.Section(nspf) for _ in range(sectionCount)] if sections[0].offset - UNCOMPRESSABLE_HEADER_SIZE > 0: fakeSection = Header.FakeSection( UNCOMPRESSABLE_HEADER_SIZE, sections[0].offset - UNCOMPRESSABLE_HEADER_SIZE) sections.insert(0, fakeSection) nca_size = UNCOMPRESSABLE_HEADER_SIZE for i in range(sectionCount): nca_size += sections[i].size pos = nspf.tell() blockMagic = nspf.read(8) nspf.seek(pos) useBlockCompression = blockMagic == b'NCZBLOCK' blockSize = -1 if useBlockCompression: BlockHeader = Header.Block(nspf) blockDecompressorReader = BlockDecompressorReader.BlockDecompressorReader( nspf, BlockHeader) pos = nspf.tell() if not useBlockCompression: decompressor = ZstdDecompressor().stream_reader(nspf) hash = sha256() if statusReportInfo == None: BAR_FMT = u'{desc}{desc_pad}{percentage:3.0f}%|{bar}| {count:{len_total}d}/{total:d} {unit} [{elapsed}<{eta}, {rate:.2f}{unit_pad}{unit}/s]' bar = enlighten.Counter(total=nca_size // 1048576, desc='Decompress', unit="MiB", color='red', bar_format=BAR_FMT) decompressedBytes = len(header) if f != None: f.write(header) if statusReportInfo != None: statusReport, id = statusReportInfo statusReport[id] = [len(header), 0, nca_size] else: bar.count = decompressedBytes // 1048576 bar.refresh() hash.update(header) firstSection = True for s in sections: i = s.offset useCrypto = s.cryptoType in (3, 4) if useCrypto: crypto = aes128.AESCTR(s.cryptoKey, s.cryptoCounter) end = s.offset + s.size if firstSection: firstSection = False uncompressedSize = UNCOMPRESSABLE_HEADER_SIZE - sections[0].offset if uncompressedSize > 0: i += uncompressedSize while i < end: if useCrypto: crypto.seek(i) chunkSz = 0x10000 if end - i > 0x10000 else end - i if useBlockCompression: inputChunk = blockDecompressorReader.read(chunkSz) else: inputChunk = decompressor.read(chunkSz) decompressor.flush() if not len(inputChunk): break if useCrypto: inputChunk = crypto.encrypt(inputChunk) if f != None: f.write(inputChunk) hash.update(inputChunk) lenInputChunk = len(inputChunk) i += lenInputChunk decompressedBytes += lenInputChunk if statusReportInfo != None: statusReport[id] = [ statusReport[id][0] + chunkSz, statusReport[id][1], nca_size ] else: bar.count = decompressedBytes // 1048576 bar.refresh() if statusReportInfo == None: bar.close() #Line break after closing the process bar is required to prevent #the next output from being on the same line as the process bar print() hexHash = hash.hexdigest() if f != None: end = f.tell() written = (end - start) return (written, hexHash) return (0, hexHash)
def __decompressNcz(nspf, f, statusReportInfo): UNCOMPRESSABLE_HEADER_SIZE = 0x4000 blockID = 0 nspf.seek(0) header = nspf.read(UNCOMPRESSABLE_HEADER_SIZE) if f is not None: start = f.tell() magic = nspf.read(8) if not magic == b'NCZSECTN': raise ValueError("No NCZSECTN found! Is this really a .ncz file?") sectionCount = nspf.readInt64() sections = [Section(nspf) for _ in range(sectionCount)] if sections[0].offset - UNCOMPRESSABLE_HEADER_SIZE > 0: fakeSection = FakeSection( UNCOMPRESSABLE_HEADER_SIZE, sections[0].offset - UNCOMPRESSABLE_HEADER_SIZE) sections.insert(0, fakeSection) nca_size = UNCOMPRESSABLE_HEADER_SIZE for i in range(sectionCount): nca_size += sections[i].size decompressor = ZstdDecompressor().stream_reader(nspf) hash = sha256() bar = Status.create(nspf.size, desc=os.path.basename(nspf._path), unit='B') # if statusReportInfo == None: # BAR_FMT = u'{desc}{desc_pad}{percentage:3.0f}%|{bar}| {count:{len_total}d}/{total:d} {unit} [{elapsed}<{eta}, {rate:.2f}{unit_pad}{unit}/s]' # bar = enlighten.Counter(total=nca_size//1048576, desc='Decompress', unit="MiB", color='red', bar_format=BAR_FMT) decompressedBytes = len(header) if f is not None: f.write(header) bar.add(len(header)) hash.update(header) firstSection = True for s in sections: i = s.offset useCrypto = s.cryptoType in (3, 4) if useCrypto: crypto = aes128.AESCTR(s.cryptoKey, s.cryptoCounter) end = s.offset + s.size if firstSection: firstSection = False uncompressedSize = UNCOMPRESSABLE_HEADER_SIZE - sections[0].offset if uncompressedSize > 0: i += uncompressedSize while i < end: if useCrypto: crypto.seek(i) chunkSz = 0x10000 if end - i > 0x10000 else end - i inputChunk = decompressor.read(chunkSz) decompressor.flush() if not len(inputChunk): break if useCrypto: inputChunk = crypto.encrypt(inputChunk) if f is not None: f.write(inputChunk) bar.add(len(inputChunk)) hash.update(inputChunk) lenInputChunk = len(inputChunk) i += lenInputChunk decompressedBytes += lenInputChunk bar.add(lenInputChunk) bar.close() print() hexHash = hash.hexdigest() if f is not None: end = f.tell() written = (end - start) return (written, hexHash) return (0, hexHash)
def __decompress(filePath, outputDir = None, write = True, raiseVerificationException = False): ncaHeaderSize = 0x4000 CHUNK_SZ = 0x100000 if write: nspPath = Path(filePath[0:-1] + 'p') if outputDir == None else Path(outputDir).joinpath(Path(filePath[0:-1] + 'p').name).resolve(strict=False) Print.info('decompressing %s -> %s' % (filePath, nspPath)) newNsp = Pfs0.Pfs0Stream(nspPath) fileHashes = FileExistingChecks.ExtractHashes(filePath) filePath = str(Path(filePath).resolve()) container = factory(filePath) container.open(filePath, 'rb') for nspf in container: if isinstance(nspf, Nca.Nca) and nspf.header.contentType == Type.Content.DATA: Print.info('skipping delta fragment') continue if not nspf._path.endswith('.ncz'): verifyFile = nspf._path.endswith('.nca') and not nspf._path.endswith('.cnmt.nca') if write: f = newNsp.add(nspf._path, nspf.size) hash = sha256() nspf.seek(0) while not nspf.eof(): inputChunk = nspf.read(CHUNK_SZ) hash.update(inputChunk) if write: f.write(inputChunk) if verifyFile: if hash.hexdigest() in fileHashes: Print.error('[VERIFIED] {0}'.format(nspf._path)) else: Print.info('[CORRUPTED] {0}'.format(nspf._path)) if raiseVerificationException: raise Exception("Verification detected hash missmatch!") elif not write: Print.info('[EXISTS] {0}'.format(nspf._path)) continue newFileName = nspf._path[0:-1] + 'a' if write: f = newNsp.add(newFileName, nspf.size) start = f.tell() blockID = 0 nspf.seek(0) header = nspf.read(ncaHeaderSize) magic = nspf.read(8) if not magic == b'NCZSECTN': raise ValueError("No NCZSECTN found! Is this really a .ncz file?") sectionCount = nspf.readInt64() sections = [Header.Section(nspf) for _ in range(sectionCount)] nca_size = ncaHeaderSize for i in range(sectionCount): nca_size += sections[i].size pos = nspf.tell() blockMagic = nspf.read(8) nspf.seek(pos) useBlockCompression = blockMagic == b'NCZBLOCK' blockSize = -1 if useBlockCompression: BlockHeader = Header.Block(nspf) blockDecompressorReader = BlockDecompressorReader.BlockDecompressorReader(nspf, BlockHeader) pos = nspf.tell() if not useBlockCompression: decompressor = ZstdDecompressor().stream_reader(nspf) hash = sha256() with tqdm(total=nca_size, unit_scale=True, unit="B") as bar: if write: f.write(header) bar.update(len(header)) hash.update(header) for s in sections: i = s.offset crypto = aes128.AESCTR(s.cryptoKey, s.cryptoCounter) end = s.offset + s.size while i < end: crypto.seek(i) chunkSz = 0x10000 if end - i > 0x10000 else end - i if useBlockCompression: inputChunk = blockDecompressorReader.read(chunkSz) else: inputChunk = decompressor.read(chunkSz) if not len(inputChunk): break if not useBlockCompression: decompressor.flush() if s.cryptoType in (3, 4): inputChunk = crypto.encrypt(inputChunk) if write: f.write(inputChunk) hash.update(inputChunk) i += len(inputChunk) bar.update(chunkSz) if hash.hexdigest() in fileHashes: Print.error('[VERIFIED] {0}'.format(nspf._path)) else: Print.info('[CORRUPTED] {0}'.format(nspf._path)) if raiseVerificationException: raise Exception("Verification detected hash missmatch") if write: end = f.tell() written = (end - start) newNsp.resize(newFileName, written) continue if write: newNsp.close() container.close()