def blockCompressContainer(readContainer, writeContainer, compressionLevel,
blockSizeExponent, threads):
CHUNK_SZ = 0x100000
UNCOMPRESSABLE_HEADER_SIZE = 0x4000
if blockSizeExponent < 14 or blockSizeExponent > 32:
raise ValueError("Block size must be between 14 and 32")
blockSize = 2**blockSizeExponent
manager = Manager()
results = manager.list()
readyForWork = Counter(0)
pleaseKillYourself = Counter(0)
TasksPerChunk = 209715200 // blockSize
for i in range(TasksPerChunk):
results.append(b"")
pool = []
work = manager.Queue(threads)
for i in range(threads):
p = Process(target=compressBlockTask,
args=(work, results, readyForWork, pleaseKillYourself))
p.start()
pool.append(p)
for nspf in readContainer:
if isinstance(
nspf,
Nca.Nca) and nspf.header.contentType == Type.Content.DATA:
Print.info('Skipping delta fragment {0}'.format(nspf._path))
continue
if isinstance(nspf, Nca.Nca) and (
nspf.header.contentType == Type.Content.PROGRAM
or nspf.header.contentType == Type.Content.PUBLICDATA
) and nspf.size > UNCOMPRESSABLE_HEADER_SIZE:
if isNcaPacked(nspf):
offsetFirstSection = sortedFs(nspf)[0].offset
newFileName = nspf._path[0:-1] + 'z'
f = writeContainer.add(newFileName, nspf.size)
startPos = f.tell()
nspf.seek(0)
f.write(nspf.read(UNCOMPRESSABLE_HEADER_SIZE))
sections = []
for fs in sortedFs(nspf):
sections += fs.getEncryptionSections()
if len(sections) == 0:
for p in pool:
#Process.terminate() might corrupt the datastructure but we do't care
p.terminate()
raise Exception(
"NCA can't be decrypted. Outdated keys.txt?")
header = b'NCZSECTN'
header += len(sections).to_bytes(8, 'little')
i = 0
for fs in sections:
i += 1
header += fs.offset.to_bytes(8, 'little')
header += fs.size.to_bytes(8, 'little')
header += fs.cryptoType.to_bytes(8, 'little')
header += b'\x00' * 8
header += fs.cryptoKey
header += fs.cryptoCounter
f.write(header)
blockID = 0
chunkRelativeBlockID = 0
startChunkBlockID = 0
blocksHeaderFilePos = f.tell()
bytesToCompress = nspf.size - UNCOMPRESSABLE_HEADER_SIZE
blocksToCompress = bytesToCompress // blockSize + (
bytesToCompress % blockSize > 0)
compressedblockSizeList = [0] * blocksToCompress
header = b'NCZBLOCK' #Magic
header += b'\x02' #Version
header += b'\x01' #Type
header += b'\x00' #Unused
header += blockSizeExponent.to_bytes(
1, 'little') #blockSizeExponent in bits: 2^x
header += blocksToCompress.to_bytes(
4, 'little') #Amount of Blocks
header += bytesToCompress.to_bytes(
8, 'little') #Decompressed Size
header += b'\x00' * (blocksToCompress * 4)
f.write(header)
decompressedBytes = UNCOMPRESSABLE_HEADER_SIZE
compressedBytes = f.tell()
BAR_FMT = u'{desc}{desc_pad}{percentage:3.0f}%|{bar}| {count:{len_total}d}/{total:d} {unit} [{elapsed}<{eta}, {rate:.2f}{unit_pad}{unit}/s]'
bar = enlighten.Counter(total=nspf.size // 1048576,
desc='Compressing',
unit='MiB',
color='cyan',
bar_format=BAR_FMT)
subBars = bar.add_subcounter('green', all_fields=True)
partitions = []
if offsetFirstSection - UNCOMPRESSABLE_HEADER_SIZE > 0:
partitions.append(
nspf.partition(offset=UNCOMPRESSABLE_HEADER_SIZE,
size=offsetFirstSection -
UNCOMPRESSABLE_HEADER_SIZE,
cryptoType=Type.Crypto.CTR.NONE,
autoOpen=True))
for section in sections:
#Print.info('offset: %x\t\tsize: %x\t\ttype: %d\t\tiv%s' % (section.offset, section.size, section.cryptoType, str(hx(section.cryptoCounter))), pleaseNoPrint)
partitions.append(
nspf.partition(offset=section.offset,
size=section.size,
cryptoType=section.cryptoType,
cryptoKey=section.cryptoKey,
cryptoCounter=bytearray(
section.cryptoCounter),
autoOpen=True))
if UNCOMPRESSABLE_HEADER_SIZE - offsetFirstSection > 0:
partitions[0].seek(UNCOMPRESSABLE_HEADER_SIZE -
offsetFirstSection)
partNr = 0
bar.count = nspf.tell() // 1048576
subBars.count = f.tell() // 1048576
bar.refresh()
while True:
buffer = partitions[partNr].read(blockSize)
while (len(buffer) < blockSize
and partNr < len(partitions) - 1):
partitions[partNr].close()
partitions[partNr] = None
partNr += 1
buffer += partitions[partNr].read(blockSize -
len(buffer))
if chunkRelativeBlockID >= TasksPerChunk or len(
buffer) == 0:
while readyForWork.value() < threads:
sleep(0.02)
for i in range(
min(TasksPerChunk,
blocksToCompress - startChunkBlockID)):
lenResult = len(results[i])
compressedBytes += lenResult
compressedblockSizeList[startChunkBlockID +
i] = lenResult
f.write(results[i])
results[i] = b""
if len(buffer) == 0:
break
chunkRelativeBlockID = 0
startChunkBlockID = blockID
work.put([
buffer, compressionLevel, compressedblockSizeList,
chunkRelativeBlockID
])
blockID += 1
chunkRelativeBlockID += 1
decompressedBytes += len(buffer)
bar.count = decompressedBytes // 1048576
subBars.count = compressedBytes // 1048576
bar.refresh()
partitions[partNr].close()
partitions[partNr] = None
endPos = f.tell()
bar.count = decompressedBytes // 1048576
subBars.count = compressedBytes // 1048576
bar.close()
written = endPos - startPos
f.seek(blocksHeaderFilePos + 24)
header = b""
for compressedblockSize in compressedblockSizeList:
header += compressedblockSize.to_bytes(4, 'little')
f.write(header)
f.seek(endPos) #Seek to end of file.
Print.info('compressed %d%% %d -> %d - %s' %
(int(written * 100 / nspf.size), decompressedBytes,
written, nspf._path))
writeContainer.resize(newFileName, written)
continue
else:
Print.info('Skipping not packed {0}'.format(nspf._path))
f = writeContainer.add(nspf._path, nspf.size)
nspf.seek(0)
while not nspf.eof():
buffer = nspf.read(CHUNK_SZ)
f.write(buffer)
#Ensures that all threads are started and compleaded before being requested to quit
while readyForWork.value() < threads:
sleep(0.02)
pleaseKillYourself.increment()
for i in range(readyForWork.value()):
work.put(None)
while readyForWork.value() > 0:
sleep(0.02)
def processContainer(readContainer, writeContainer, compressionLevel, threads,
stusReport, id, pleaseNoPrint):
for nspf in readContainer:
if isinstance(
nspf,
Nca.Nca) and nspf.header.contentType == Type.Content.DATA:
Print.info('Skipping delta fragment {0}'.format(nspf._path))
continue
if isinstance(nspf, Nca.Nca) and (
nspf.header.contentType == Type.Content.PROGRAM
or nspf.header.contentType == Type.Content.PUBLICDATA
) and nspf.size > UNCOMPRESSABLE_HEADER_SIZE:
if isNcaPacked(nspf):
offsetFirstSection = sortedFs(nspf)[0].offset
newFileName = nspf._path[0:-1] + 'z'
with writeContainer.add(newFileName, nspf.size,
pleaseNoPrint) as f:
start = f.tell()
nspf.seek(0)
f.write(nspf.read(UNCOMPRESSABLE_HEADER_SIZE))
sections = []
for fs in sortedFs(nspf):
sections += fs.getEncryptionSections()
if len(sections) == 0:
raise Exception(
"NCA can't be decrypted. Outdated keys.txt?")
header = b'NCZSECTN'
header += len(sections).to_bytes(8, 'little')
i = 0
for fs in sections:
i += 1
header += fs.offset.to_bytes(8, 'little')
header += fs.size.to_bytes(8, 'little')
header += fs.cryptoType.to_bytes(8, 'little')
header += b'\x00' * 8
header += fs.cryptoKey
header += fs.cryptoCounter
f.write(header)
blockID = 0
chunkRelativeBlockID = 0
startChunkBlockID = 0
blocksHeaderFilePos = f.tell()
compressedblockSizeList = []
decompressedBytes = UNCOMPRESSABLE_HEADER_SIZE
stusReport[id] = [0, 0, nspf.size]
partitions = []
if offsetFirstSection - UNCOMPRESSABLE_HEADER_SIZE > 0:
partitions.append(
nspf.partition(offset=UNCOMPRESSABLE_HEADER_SIZE,
size=offsetFirstSection -
UNCOMPRESSABLE_HEADER_SIZE,
cryptoType=Type.Crypto.CTR.NONE,
autoOpen=True))
for section in sections:
#Print.info('offset: %x\t\tsize: %x\t\ttype: %d\t\tiv%s' % (section.offset, section.size, section.cryptoType, str(hx(section.cryptoCounter))), pleaseNoPrint)
partitions.append(
nspf.partition(offset=section.offset,
size=section.size,
cryptoType=section.cryptoType,
cryptoKey=section.cryptoKey,
cryptoCounter=bytearray(
section.cryptoCounter),
autoOpen=True))
if UNCOMPRESSABLE_HEADER_SIZE - offsetFirstSection > 0:
partitions[0].seek(UNCOMPRESSABLE_HEADER_SIZE -
offsetFirstSection)
partNr = 0
stusReport[id] = [nspf.tell(), f.tell(), nspf.size]
if threads > 1:
cctx = ZstdCompressor(level=compressionLevel,
threads=threads)
else:
cctx = ZstdCompressor(level=compressionLevel)
compressor = cctx.stream_writer(f)
while True:
buffer = partitions[partNr].read(CHUNK_SZ)
while (len(buffer) < CHUNK_SZ
and partNr < len(partitions) - 1):
partitions[partNr].close()
partitions[partNr] = None
partNr += 1
buffer += partitions[partNr].read(CHUNK_SZ -
len(buffer))
if len(buffer) == 0:
break
compressor.write(buffer)
decompressedBytes += len(buffer)
stusReport[id] = [nspf.tell(), f.tell(), nspf.size]
partitions[partNr].close()
partitions[partNr] = None
compressor.flush(FLUSH_FRAME)
compressor.flush(COMPRESSOBJ_FLUSH_FINISH)
stusReport[id] = [nspf.tell(), f.tell(), nspf.size]
written = f.tell() - start
Print.info(
'Compressed {0}% {1} -> {2} - {3}'.format(
written * 100 / nspf.size, decompressedBytes,
written, nspf._path), pleaseNoPrint)
writeContainer.resize(newFileName, written)
continue
else:
Print.info('Skipping not packed {0}'.format(nspf._path))
with writeContainer.add(nspf._path, nspf.size, pleaseNoPrint) as f:
nspf.seek(0)
while not nspf.eof():
buffer = nspf.read(CHUNK_SZ)
f.write(buffer)