def setupCrypto(self, cryptoType = -1, cryptoKey = -1, cryptoCounter = -1):
if cryptoType != -1:
self.cryptoType = cryptoType
if cryptoKey != -1:
self.cryptoKey = cryptoKey
if cryptoCounter != -1:
self.cryptoCounter = cryptoCounter
if self.cryptoType == Fs.Type.Crypto.CTR:
if self.cryptoKey:
self.crypto = aes128.AESCTR(self.cryptoKey, self.setCounter(self.offset))
self.cryptoType = Fs.Type.Crypto.CTR
self.enableBufferedIO(0x10, 0x10)
elif self.cryptoType == Fs.Type.Crypto.XTS:
if self.cryptoKey:
self.crypto = aes128.AESXTS(self.cryptoKey)
self.cryptoType = Fs.Type.Crypto.XTS
if self.size < 1 or self.size > 0xFFFFFF:
raise IOError('AESXTS Block too large or small')
self.rewind()
self.enableBufferedIO(self.size, 0x10)
elif self.cryptoType == Fs.Type.Crypto.BKTR:
self.cryptoType = Fs.Type.Crypto.BKTR
elif self.cryptoType == Fs.Type.Crypto.NCA0:
self.cryptoType = Fs.Type.Crypto.NCA0
elif self.cryptoType == Fs.Type.Crypto.NONE:
self.cryptoType = Fs.Type.Crypto.NONE
def __decompressNcz(nspf, f, statusReportInfo, pleaseNoPrint):
ncaHeaderSize = 0x4000
blockID = 0
nspf.seek(0)
header = nspf.read(ncaHeaderSize)
if f != None:
start = f.tell()
magic = nspf.read(8)
if not magic == b'NCZSECTN':
raise ValueError("No NCZSECTN found! Is this really a .ncz file?")
sectionCount = nspf.readInt64()
sections = [Header.Section(nspf) for _ in range(sectionCount)]
nca_size = ncaHeaderSize
for i in range(sectionCount):
nca_size += sections[i].size
pos = nspf.tell()
blockMagic = nspf.read(8)
nspf.seek(pos)
useBlockCompression = blockMagic == b'NCZBLOCK'
blockSize = -1
if useBlockCompression:
BlockHeader = Header.Block(nspf)
blockDecompressorReader = BlockDecompressorReader.BlockDecompressorReader(
nspf, BlockHeader)
pos = nspf.tell()
if not useBlockCompression:
decompressor = ZstdDecompressor().stream_reader(nspf)
hash = sha256()
if statusReportInfo == None:
BAR_FMT = u'{desc}{desc_pad}{percentage:3.0f}%|{bar}| {count:{len_total}d}/{total:d} {unit} [{elapsed}<{eta}, {rate:.2f}{unit_pad}{unit}/s]'
bar = enlighten.Counter(total=nca_size // 1048576,
desc='Decompress',
unit="MiB",
color='red',
bar_format=BAR_FMT)
decompressedBytes = len(header)
if f != None:
f.write(header)
if statusReportInfo != None:
statusReport, id = statusReportInfo
statusReport[id] = [len(header), 0, nca_size]
else:
bar.count = decompressedBytes // 1048576
bar.refresh()
hash.update(header)
for s in sections:
i = s.offset
crypto = aes128.AESCTR(s.cryptoKey, s.cryptoCounter)
end = s.offset + s.size
while i < end:
crypto.seek(i)
chunkSz = 0x10000 if end - i > 0x10000 else end - i
if useBlockCompression:
inputChunk = blockDecompressorReader.read(chunkSz)
else:
inputChunk = decompressor.read(chunkSz)
if not len(inputChunk):
break
if not useBlockCompression:
decompressor.flush()
if s.cryptoType in (3, 4):
inputChunk = crypto.encrypt(inputChunk)
if f != None:
f.write(inputChunk)
hash.update(inputChunk)
lenInputChunk = len(inputChunk)
i += lenInputChunk
decompressedBytes += lenInputChunk
if statusReportInfo != None:
statusReport[id] = [
statusReport[id][0] + chunkSz, statusReport[id][1],
nca_size
]
else:
bar.count = decompressedBytes // 1048576
bar.refresh()
bar.close()
hexHash = hash.hexdigest()
if f != None:
end = f.tell()
written = (end - start)
return (written, hexHash)
return (0, hexHash)
def __decompressNcz(nspf, f, statusReportInfo, pleaseNoPrint):
UNCOMPRESSABLE_HEADER_SIZE = 0x4000
blockID = 0
nspf.seek(0)
header = nspf.read(UNCOMPRESSABLE_HEADER_SIZE)
if f != None:
start = f.tell()
magic = nspf.read(8)
if not magic == b'NCZSECTN':
raise ValueError("No NCZSECTN found! Is this really a .ncz file?")
sectionCount = nspf.readInt64()
sections = [Header.Section(nspf) for _ in range(sectionCount)]
if sections[0].offset-UNCOMPRESSABLE_HEADER_SIZE > 0:
fakeSection = Header.FakeSection(UNCOMPRESSABLE_HEADER_SIZE, sections[0].offset-UNCOMPRESSABLE_HEADER_SIZE)
sections.insert(0, fakeSection)
nca_size = UNCOMPRESSABLE_HEADER_SIZE
for i in range(sectionCount):
nca_size += sections[i].size
pos = nspf.tell()
blockMagic = nspf.read(8)
nspf.seek(pos)
useBlockCompression = blockMagic == b'NCZBLOCK'
blockSize = -1
if useBlockCompression:
BlockHeader = Header.Block(nspf)
blockDecompressorReader = BlockDecompressorReader.BlockDecompressorReader(nspf, BlockHeader)
pos = nspf.tell()
if not useBlockCompression:
decompressor = ZstdDecompressor().stream_reader(nspf)
hash = sha256()
if statusReportInfo == None:
BAR_FMT = u'{desc}{desc_pad}{percentage:3.0f}%|{bar}| {count:{len_total}d}/{total:d} {unit} [{elapsed}<{eta}, {rate:.2f}{unit_pad}{unit}/s]'
bar = enlighten.Counter(total=nca_size//1048576, desc='Decompress', unit="MiB", color='red', bar_format=BAR_FMT)
decompressedBytes = len(header)
if f != None:
f.write(header)
if statusReportInfo != None:
statusReport, id = statusReportInfo
statusReport[id] = [len(header), 0, nca_size]
else:
bar.count = decompressedBytes//1048576
bar.refresh()
hash.update(header)
firstSection = True
for s in sections:
i = s.offset
useCrypto = s.cryptoType in (3, 4)
if useCrypto:
crypto = aes128.AESCTR(s.cryptoKey, s.cryptoCounter)
end = s.offset + s.size
if firstSection:
firstSection = False
uncompressedSize = UNCOMPRESSABLE_HEADER_SIZE-sections[0].offset
if uncompressedSize > 0:
i += uncompressedSize
while i < end:
if useCrypto:
crypto.seek(i)
chunkSz = 0x10000 if end - i > 0x10000 else end - i
if useBlockCompression:
inputChunk = blockDecompressorReader.read(chunkSz)
else:
inputChunk = decompressor.read(chunkSz)
decompressor.flush()
if not len(inputChunk):
break
if useCrypto:
inputChunk = crypto.encrypt(inputChunk)
if f != None:
f.write(inputChunk)
hash.update(inputChunk)
lenInputChunk = len(inputChunk)
i += lenInputChunk
decompressedBytes += lenInputChunk
if statusReportInfo != None:
statusReport[id] = [statusReport[id][0]+chunkSz, statusReport[id][1], nca_size]
else:
bar.count = decompressedBytes//1048576
bar.refresh()
if statusReportInfo == None:
bar.close()
#Line break after closing the process bar is required to prevent
#the next output from being on the same line as the process bar
print()
hexHash = hash.hexdigest()
if f != None:
end = f.tell()
written = (end - start)
return (written, hexHash)
return (0, hexHash)
def __decompressNcz(nspf, f, statusReportInfo):
UNCOMPRESSABLE_HEADER_SIZE = 0x4000
blockID = 0
nspf.seek(0)
header = nspf.read(UNCOMPRESSABLE_HEADER_SIZE)
if f is not None:
start = f.tell()
magic = nspf.read(8)
if not magic == b'NCZSECTN':
raise ValueError("No NCZSECTN found! Is this really a .ncz file?")
sectionCount = nspf.readInt64()
sections = [Section(nspf) for _ in range(sectionCount)]
if sections[0].offset - UNCOMPRESSABLE_HEADER_SIZE > 0:
fakeSection = FakeSection(
UNCOMPRESSABLE_HEADER_SIZE,
sections[0].offset - UNCOMPRESSABLE_HEADER_SIZE)
sections.insert(0, fakeSection)
nca_size = UNCOMPRESSABLE_HEADER_SIZE
for i in range(sectionCount):
nca_size += sections[i].size
decompressor = ZstdDecompressor().stream_reader(nspf)
hash = sha256()
bar = Status.create(nspf.size, desc=os.path.basename(nspf._path), unit='B')
# if statusReportInfo == None:
# BAR_FMT = u'{desc}{desc_pad}{percentage:3.0f}%|{bar}| {count:{len_total}d}/{total:d} {unit} [{elapsed}<{eta}, {rate:.2f}{unit_pad}{unit}/s]'
# bar = enlighten.Counter(total=nca_size//1048576, desc='Decompress', unit="MiB", color='red', bar_format=BAR_FMT)
decompressedBytes = len(header)
if f is not None:
f.write(header)
bar.add(len(header))
hash.update(header)
firstSection = True
for s in sections:
i = s.offset
useCrypto = s.cryptoType in (3, 4)
if useCrypto:
crypto = aes128.AESCTR(s.cryptoKey, s.cryptoCounter)
end = s.offset + s.size
if firstSection:
firstSection = False
uncompressedSize = UNCOMPRESSABLE_HEADER_SIZE - sections[0].offset
if uncompressedSize > 0:
i += uncompressedSize
while i < end:
if useCrypto:
crypto.seek(i)
chunkSz = 0x10000 if end - i > 0x10000 else end - i
inputChunk = decompressor.read(chunkSz)
decompressor.flush()
if not len(inputChunk):
break
if useCrypto:
inputChunk = crypto.encrypt(inputChunk)
if f is not None:
f.write(inputChunk)
bar.add(len(inputChunk))
hash.update(inputChunk)
lenInputChunk = len(inputChunk)
i += lenInputChunk
decompressedBytes += lenInputChunk
bar.add(lenInputChunk)
bar.close()
print()
hexHash = hash.hexdigest()
if f is not None:
end = f.tell()
written = (end - start)
return (written, hexHash)
return (0, hexHash)
def __decompress(filePath, outputDir = None, write = True, raiseVerificationException = False):
ncaHeaderSize = 0x4000
CHUNK_SZ = 0x100000
if write:
if outputDir is None:
nspPath = filePath[0:-1] + 'p'
else:
nspPath = os.path.join(outputDir, os.path.basename(filePath[0:-1] + 'p'))
nspPath = os.path.abspath(nspPath)
Print.info('decompressing %s -> %s' % (filePath, nspPath))
newNsp = Fs.Pfs0.Pfs0Stream(nspPath)
filePath = os.path.abspath(filePath)
container = Fs.factory(filePath)
container.open(filePath, 'rb')
for nspf in container:
if isinstance(nspf, Fs.Nca.Nca) and nspf.header.contentType == Fs.Type.Content.DATA:
Print.info('skipping delta fragment')
continue
if not nspf._path.endswith('.ncz'):
verifyFile = nspf._path.endswith('.nca') and not nspf._path.endswith('.cnmt.nca')
if write:
f = newNsp.add(nspf._path, nspf.size)
hash = hashlib.sha256()
nspf.seek(0)
while not nspf.eof():
inputChunk = nspf.read(CHUNK_SZ)
hash.update(inputChunk)
if write:
f.write(inputChunk)
hexHash = hash.hexdigest()[0:32]
if verifyFile:
if hexHash + '.nca' == nspf._path:
Print.error('[VERIFIED] {0}'.format(nspf._path))
else:
Print.info('[CORRUPTED] {0}'.format(nspf._path))
if raiseVerificationException:
raise Exception("Verification detected hash missmatch!")
elif not write:
Print.info('[EXISTS] {0}'.format(nspf._path))
continue
newFileName = nspf._path[0:-1] + 'a'
if write:
f = newNsp.add(newFileName, nspf.size)
start = f.tell()
blockID = 0
nspf.seek(0)
header = nspf.read(ncaHeaderSize)
magic = nspf.read(8)
if not magic == b'NCZSECTN':
raise ValueError("No NCZSECTN found! Is this really a .ncz file?")
sectionCount = nspf.readInt64()
sections = []
for i in range(sectionCount):
sections.append(Header.Section(nspf))
pos = nspf.tell()
blockMagic = nspf.read(8)
nspf.seek(pos)
useBlockCompression = blockMagic == b'NCZBLOCK'
blockSize = -1
if useBlockCompression:
BlockHeader = Header.Block(nspf)
blockDecompressorReader = BlockDecompressorReader.BlockDecompressorReader(nspf, BlockHeader)
pos = nspf.tell()
dctx = zstandard.ZstdDecompressor()
if not useBlockCompression:
decompressor = dctx.stream_reader(nspf)
hash = hashlib.sha256()
with tqdm(total=nspf.size, unit_scale=True, unit="B/s") as bar:
if write:
f.write(header)
bar.update(len(header))
hash.update(header)
for s in sections:
i = s.offset
crypto = aes128.AESCTR(s.cryptoKey, s.cryptoCounter)
end = s.offset + s.size
while i < end:
crypto.seek(i)
chunkSz = 0x10000 if end - i > 0x10000 else end - i
if useBlockCompression:
inputChunk = blockDecompressorReader.read(chunkSz)
else:
inputChunk = decompressor.read(chunkSz)
if not len(inputChunk):
break
if not useBlockCompression:
decompressor.flush()
if s.cryptoType in (3, 4):
inputChunk = crypto.encrypt(inputChunk)
if write:
f.write(inputChunk)
bar.update(len(inputChunk))
hash.update(inputChunk)
i += len(inputChunk)
hexHash = hash.hexdigest()[0:32]
if hexHash + '.nca' == newFileName:
Print.error('[VERIFIED] {0}'.format(nspf._path))
else:
Print.info('[CORRUPTED] {0}'.format(nspf._path))
if raiseVerificationException:
raise Exception("Verification detected hash missmatch")
if write:
end = f.tell()
written = (end - start)
newNsp.resize(newFileName, written)
continue
if write:
newNsp.close()
def decompress(filePath, outputDir = None):
filePath = os.path.abspath(filePath)
container = Fs.factory(filePath)
container.open(filePath, 'rb')
CHUNK_SZ = 0x1000000
if outputDir is None:
nspPath = filePath[0:-1] + 'p'
else:
nspPath = os.path.join(outputDir, os.path.basename(filePath[0:-1] + 'p'))
nspPath = os.path.abspath(nspPath)
Print.info('decompressing %s -> %s' % (filePath, nspPath))
newNsp = Fs.Pfs0.Pfs0Stream(nspPath)
for nspf in container:
if isinstance(nspf, Fs.Nca.Nca) and nspf.header.contentType == Fs.Type.Content.DATA:
Print.info('skipping delta fragment')
continue
if nspf._path.endswith('.ncz'):
newFileName = nspf._path[0:-1] + 'a'
f = newNsp.add(newFileName, nspf.size)
start = nspf.tell()
nspf.seek(0)
header = nspf.read(ncaHeaderSize)
magic = nspf.read(8)
if not magic == b'NCZSECTN':
raise ValueError("No NCZSECTN found! Is this really a .ncz file?")
sectionCount = nspf.readInt64()
sections = []
for i in range(sectionCount):
sections.append(Section(nspf))
pos = nspf.tell()
blockMagic = nspf.read(8)
nspf.seek(pos)
useBlockCompression = blockMagic == b'NCZBLOCK'
blockSize = -1
if useBlockCompression:
BlockHeader = Block(nspf)
if BlockHeader.blockSizeExponent < 14 or BlockHeader.blockSizeExponent > 32:
raise ValueError("Corrupted NCZBLOCK header: Block size must be between 14 and 32")
blockSize = 2**BlockHeader.blockSizeExponent
pos = nspf.tell()
dctx = zstandard.ZstdDecompressor()
if not useBlockCompression:
decompressor = dctx.stream_reader(nspf)
hash = hashlib.sha256()
with tqdm(total=nspf.size, unit_scale=True, unit="B/s") as bar:
f.write(header)
bar.update(len(header))
hash.update(header)
for s in sections:
if s.cryptoType == 1: #plain text
continue
if s.cryptoType not in (3, 4):
raise IOError('unknown crypto type: %d' % s.cryptoType)
i = s.offset
crypto = aes128.AESCTR(s.cryptoKey, s.cryptoCounter)
end = s.offset + s.size
while i < end:
#f.seek(i)
crypto.seek(i)
if useBlockCompression:
decompressor = dctx.stream_reader(nspf)
inputChunk = decompressor.read(blockSize)
decompressedBytes += len(inputChunk)
o.write(inputChunk)
decompressor.flush()
o.flush()
print('Block', str(blockID+1)+'/'+str(BlockHeader.numberOfBlocks))
pos += BlockHeader.compressedBlockSizeList[blockID]
nspf.seek(pos)
blockID += 1
if(blockID >= len(BlockHeader.compressedBlockSizeList)):
break
else:
chunkSz = 0x10000 if end - i > 0x10000 else end - i
buf = decompressor.read(chunkSz)
if not len(buf):
break
#f.seek(i)
buf = crypto.encrypt(buf)
f.write(buf)
bar.update(len(buf))
hash.update(buf)
i += chunkSz
if useBlockCompression and not decompressedBytes == BlockHeader.decompressedSize:
Print.error("\nSomething went wrong! decompressedBytes != BlockHeader.decompressedSize:", decompressedBytes, "vs.", BlockHeader.decompressedSize)
hexHash = hash.hexdigest()[0:32]
if hexHash + '.nca' != newFileName:
print(hexHash + '.nca')
print(newFileName)
Print.error('\nNCZ verification failed!\n')
else:
Print.info('\nNCZ verification successful!\n')
end = f.tell()
written = end - start
newNsp.resize(newFileName, written)
continue
f = newNsp.add(nspf._path, nspf.size)
nspf.seek(0)
while not nspf.eof():
buffer = nspf.read(CHUNK_SZ)
f.write(buffer)
newNsp.close()
