def processContainer(readContainer, writeContainer, compressionLevel, threads,
stusReport, id, pleaseNoPrint):
for nspf in readContainer:
if isinstance(
nspf,
Nca.Nca) and nspf.header.contentType == Type.Content.DATA:
Print.info('skipping delta fragment', pleaseNoPrint)
continue
if isinstance(nspf, Nca.Nca) and (
nspf.header.contentType == Type.Content.PROGRAM
or nspf.header.contentType == Type.Content.PUBLICDATA):
if isNcaPacked(nspf, ncaHeaderSize):
newFileName = nspf._path[0:-1] + 'z'
with writeContainer.add(newFileName, nspf.size,
pleaseNoPrint) as f:
start = f.tell()
nspf.seek(0)
f.write(nspf.read(ncaHeaderSize))
sections = []
for fs in sortedFs(nspf):
sections += fs.getEncryptionSections()
if len(sections) == 0:
raise Exception(
"NCA can't be decrypted. Outdated keys.txt?")
header = b'NCZSECTN'
header += len(sections).to_bytes(8, 'little')
i = 0
for fs in sections:
i += 1
header += fs.offset.to_bytes(8, 'little')
header += fs.size.to_bytes(8, 'little')
header += fs.cryptoType.to_bytes(8, 'little')
header += b'\x00' * 8
header += fs.cryptoKey
header += fs.cryptoCounter
f.write(header)
blockID = 0
chunkRelativeBlockID = 0
startChunkBlockID = 0
blocksHeaderFilePos = f.tell()
compressedblockSizeList = []
decompressedBytes = ncaHeaderSize
stusReport[id] = [0, 0, nspf.size]
partitions = []
for section in sections:
#Print.info('offset: %x\t\tsize: %x\t\ttype: %d\t\tiv%s' % (section.offset, section.size, section.cryptoType, str(hx(section.cryptoCounter))), pleaseNoPrint)
partitions.append(
nspf.partition(offset=section.offset,
size=section.size,
n=None,
cryptoType=section.cryptoType,
cryptoKey=section.cryptoKey,
cryptoCounter=bytearray(
section.cryptoCounter),
autoOpen=True))
partNr = 0
stusReport[id] = [nspf.tell(), f.tell(), nspf.size]
if threads > 1:
cctx = ZstdCompressor(level=compressionLevel,
threads=threads)
else:
cctx = ZstdCompressor(level=compressionLevel)
compressor = cctx.stream_writer(f)
while True:
buffer = partitions[partNr].read(CHUNK_SZ)
while (len(buffer) < CHUNK_SZ
and partNr < len(partitions) - 1):
partitions[partNr].close()
partitions[partNr] = None
partNr += 1
buffer += partitions[partNr].read(CHUNK_SZ -
len(buffer))
if len(buffer) == 0:
break
compressor.write(buffer)
decompressedBytes += len(buffer)
stusReport[id] = [nspf.tell(), f.tell(), nspf.size]
partitions[partNr].close()
partitions[partNr] = None
compressor.flush(FLUSH_FRAME)
compressor.flush(COMPRESSOBJ_FLUSH_FINISH)
stusReport[id] = [nspf.tell(), f.tell(), nspf.size]
written = f.tell() - start
Print.info(
'Compressed {0}% {1} -> {2} - {3}'.format(
written * 100 / nspf.size, decompressedBytes,
written, nspf._path), pleaseNoPrint)
writeContainer.resize(newFileName, written)
continue
else:
Print.info('not packed!', pleaseNoPrint)
with writeContainer.add(nspf._path, nspf.size, pleaseNoPrint) as f:
nspf.seek(0)
while not nspf.eof():
buffer = nspf.read(CHUNK_SZ)
f.write(buffer)
def solidCompress(filePath, compressionLevel=18, outputDir=None, threads=-1):
ncaHeaderSize = 0x4000
filePath = str(Path(filePath).resolve())
container = factory(filePath)
container.open(filePath, 'rb')
CHUNK_SZ = 0x1000000
nszPath = str(
Path(filePath[0:-1] + 'z' if outputDir == None else Path(outputDir).
joinpath(Path(filePath[0:-1] + 'z').name)).resolve(strict=False))
for nspf in container:
if isinstance(nspf, Ticket.Ticket):
nspf.getRightsId()
break # No need to go for other objects
Print.info('compressing (level %d) %s -> %s' %
(compressionLevel, filePath, nszPath))
newNsp = Pfs0.Pfs0Stream(nszPath)
try:
for nspf in container:
if isinstance(
nspf,
Nca.Nca) and nspf.header.contentType == Type.Content.DATA:
Print.info('skipping delta fragment')
continue
if isinstance(nspf, Nca.Nca) and (
nspf.header.contentType == Type.Content.PROGRAM
or nspf.header.contentType == Type.Content.PUBLICDATA):
if isNcaPacked(nspf, ncaHeaderSize):
newFileName = nspf._path[0:-1] + 'z'
f = newNsp.add(newFileName, nspf.size)
start = f.tell()
nspf.seek(0)
f.write(nspf.read(ncaHeaderSize))
sections = []
for fs in sortedFs(nspf):
sections += fs.getEncryptionSections()
if len(sections) == 0:
raise Exception(
"NCA can't be decrypted. Outdated keys.txt?")
header = b'NCZSECTN'
header += len(sections).to_bytes(8, 'little')
for fs in sections:
header += fs.offset.to_bytes(8, 'little')
header += fs.size.to_bytes(8, 'little')
header += fs.cryptoType.to_bytes(8, 'little')
header += b'\x00' * 8
header += fs.cryptoKey
header += fs.cryptoCounter
f.write(header)
decompressedBytes = ncaHeaderSize
with tqdm(total=nspf.size, unit_scale=True,
unit="B") as bar:
partitions = [
nspf.partition(
offset=section.offset,
size=section.size,
n=None,
cryptoType=section.cryptoType,
cryptoKey=section.cryptoKey,
cryptoCounter=bytearray(section.cryptoCounter),
autoOpen=True) for section in sections
]
partNr = 0
bar.update(f.tell())
cctx = ZstdCompressor(
level=compressionLevel, threads=threads
) if threads > 1 else ZstdCompressor(
level=compressionLevel)
compressor = cctx.stream_writer(f)
while True:
buffer = partitions[partNr].read(CHUNK_SZ)
while (len(buffer) < CHUNK_SZ
and partNr < len(partitions) - 1):
partitions[partNr].close()
partitions[partNr] = None
partNr += 1
buffer += partitions[partNr].read(CHUNK_SZ -
len(buffer))
if len(buffer) == 0:
break
compressor.write(buffer)
decompressedBytes += len(buffer)
bar.update(len(buffer))
partitions[partNr].close()
partitions[partNr] = None
compressor.flush(FLUSH_FRAME)
compressor.flush(COMPRESSOBJ_FLUSH_FINISH)
written = f.tell() - start
print('compressed %d%% %d -> %d - %s' %
(int(written * 100 / nspf.size), decompressedBytes,
written, nspf._path))
newNsp.resize(newFileName, written)
continue
else:
print('not packed!')
f = newNsp.add(nspf._path, nspf.size)
nspf.seek(0)
while not nspf.eof():
buffer = nspf.read(CHUNK_SZ)
f.write(buffer)
except KeyboardInterrupt:
remove(nszPath)
raise KeyboardInterrupt
except BaseException:
Print.error(format_exc())
remove(nszPath)
finally:
newNsp.close()
container.close()
return nszPath
def blockCompress(filePath,
compressionLevel=18,
blockSizeExponent=20,
outputDir=None,
threads=32):
CHUNK_SZ = 0x100000
ncaHeaderSize = 0x4000
if blockSizeExponent < 14 or blockSizeExponent > 32:
raise ValueError("Block size must be between 14 and 32")
blockSize = 2**blockSizeExponent
filePath = str(Path(filePath).resolve(strict=False))
container = factory(filePath)
container.open(filePath, 'rb')
nszPath = str(
Path(filePath[0:-1] + 'z' if outputDir == None else str(
Path(outputDir).joinpath(Path(filePath[0:-1] +
'z').name))).resolve(strict=False))
Print.info('compressing (level %d) %s -> %s' %
(compressionLevel, filePath, nszPath))
newNsp = Pfs0.Pfs0Stream(nszPath)
try:
manager = Manager()
results = manager.list()
readyForWork = Counter(0)
pleaseKillYourself = Counter(0)
TasksPerChunk = 209715200 // blockSize
for i in range(TasksPerChunk):
results.append(b"")
work = manager.Queue(threads)
pool = []
for i in range(threads):
p = Process(target=compressBlockTask,
args=(work, results, readyForWork, pleaseKillYourself))
p.start()
pool.append(p)
for nspf in container:
if isinstance(
nspf,
Nca.Nca) and nspf.header.contentType == Type.Content.DATA:
Print.info('skipping delta fragment')
continue
if isinstance(nspf, Nca.Nca) and (
nspf.header.contentType == Type.Content.PROGRAM
or nspf.header.contentType == Type.Content.PUBLICDATA):
if isNcaPacked(nspf, ncaHeaderSize):
newFileName = nspf._path[0:-1] + 'z'
f = newNsp.add(newFileName, nspf.size)
start = f.tell()
nspf.seek(0)
f.write(nspf.read(ncaHeaderSize))
sections = []
for fs in sortedFs(nspf):
sections += fs.getEncryptionSections()
if len(sections) == 0:
raise Exception(
"NCA can't be decrypted. Outdated keys.txt?")
header = b'NCZSECTN'
header += len(sections).to_bytes(8, 'little')
i = 0
for fs in sections:
i += 1
header += fs.offset.to_bytes(8, 'little')
header += fs.size.to_bytes(8, 'little')
header += fs.cryptoType.to_bytes(8, 'little')
header += b'\x00' * 8
header += fs.cryptoKey
header += fs.cryptoCounter
f.write(header)
blockID = 0
chunkRelativeBlockID = 0
startChunkBlockID = 0
blocksHeaderFilePos = f.tell()
bytesToCompress = nspf.size - ncaHeaderSize
blocksToCompress = bytesToCompress // blockSize + (
bytesToCompress % blockSize > 0)
compressedblockSizeList = [0] * blocksToCompress
header = b'NCZBLOCK' #Magic
header += b'\x02' #Version
header += b'\x01' #Type
header += b'\x00' #Unused
header += blockSizeExponent.to_bytes(
1, 'little') #blockSizeExponent in bits: 2^x
header += blocksToCompress.to_bytes(
4, 'little') #Amount of Blocks
header += bytesToCompress.to_bytes(
8, 'little') #Decompressed Size
header += b'\x00' * (blocksToCompress * 4)
f.write(header)
decompressedBytes = ncaHeaderSize
with tqdm(total=nspf.size, unit_scale=True,
unit="B") as bar:
partitions = [
nspf.partition(
offset=section.offset,
size=section.size,
n=None,
cryptoType=section.cryptoType,
cryptoKey=section.cryptoKey,
cryptoCounter=bytearray(section.cryptoCounter),
autoOpen=True) for section in sections
]
partNr = 0
bar.update(f.tell())
while True:
buffer = partitions[partNr].read(blockSize)
while (len(buffer) < blockSize
and partNr < len(partitions) - 1):
partitions[partNr].close()
partitions[partNr] = None
partNr += 1
buffer += partitions[partNr].read(blockSize -
len(buffer))
if chunkRelativeBlockID >= TasksPerChunk or len(
buffer) == 0:
while readyForWork.value() < threads:
sleep(0.02)
for i in range(
min(
TasksPerChunk, blocksToCompress -
startChunkBlockID)):
compressedblockSizeList[startChunkBlockID +
i] = len(
results[i])
f.write(results[i])
results[i] = b""
if len(buffer) == 0:
pleaseKillYourself.increment()
for i in range(readyForWork.value()):
work.put(None)
while readyForWork.value() > 0:
sleep(0.02)
break
chunkRelativeBlockID = 0
startChunkBlockID = blockID
work.put([
buffer, compressionLevel,
compressedblockSizeList, chunkRelativeBlockID
])
blockID += 1
chunkRelativeBlockID += 1
decompressedBytes += len(buffer)
bar.update(len(buffer))
partitions[partNr].close()
partitions[partNr] = None
f.seek(blocksHeaderFilePos + 24)
header = b""
for compressedblockSize in compressedblockSizeList:
header += compressedblockSize.to_bytes(4, 'little')
f.write(header)
f.seek(0, 2) #Seek to end of file.
written = f.tell() - start
print('compressed %d%% %d -> %d - %s' %
(int(written * 100 / nspf.size), decompressedBytes,
written, nspf._path))
newNsp.resize(newFileName, written)
continue
else:
print('not packed!')
f = newNsp.add(nspf._path, nspf.size)
nspf.seek(0)
while not nspf.eof():
buffer = nspf.read(CHUNK_SZ)
f.write(buffer)
except KeyboardInterrupt:
remove(nszPath)
raise KeyboardInterrupt
except BaseException:
Print.error(format_exc())
remove(nszPath)
finally:
newNsp.close()
container.close()
return nszPath
def blockCompressContainer(readContainer, writeContainer, compressionLevel,
blockSizeExponent, threads):
CHUNK_SZ = 0x100000
ncaHeaderSize = 0x4000
if blockSizeExponent < 14 or blockSizeExponent > 32:
raise ValueError("Block size must be between 14 and 32")
blockSize = 2**blockSizeExponent
manager = Manager()
results = manager.list()
readyForWork = Counter(0)
pleaseKillYourself = Counter(0)
TasksPerChunk = 209715200 // blockSize
for i in range(TasksPerChunk):
results.append(b"")
pool = []
work = manager.Queue(threads)
for i in range(threads):
p = Process(target=compressBlockTask,
args=(work, results, readyForWork, pleaseKillYourself))
p.start()
pool.append(p)
for nspf in readContainer:
if isinstance(
nspf,
Nca.Nca) and nspf.header.contentType == Type.Content.DATA:
Print.info('skipping delta fragment')
continue
if isinstance(nspf, Nca.Nca) and (
nspf.header.contentType == Type.Content.PROGRAM
or nspf.header.contentType == Type.Content.PUBLICDATA):
if isNcaPacked(nspf, ncaHeaderSize):
newFileName = nspf._path[0:-1] + 'z'
f = writeContainer.add(newFileName, nspf.size)
startPos = f.tell()
nspf.seek(0)
f.write(nspf.read(ncaHeaderSize))
sections = []
for fs in sortedFs(nspf):
sections += fs.getEncryptionSections()
if len(sections) == 0:
for p in pool:
#Process.terminate() might corrupt the datastructure but we do't care
p.terminate()
raise Exception(
"NCA can't be decrypted. Outdated keys.txt?")
header = b'NCZSECTN'
header += len(sections).to_bytes(8, 'little')
i = 0
for fs in sections:
i += 1
header += fs.offset.to_bytes(8, 'little')
header += fs.size.to_bytes(8, 'little')
header += fs.cryptoType.to_bytes(8, 'little')
header += b'\x00' * 8
header += fs.cryptoKey
header += fs.cryptoCounter
f.write(header)
blockID = 0
chunkRelativeBlockID = 0
startChunkBlockID = 0
blocksHeaderFilePos = f.tell()
bytesToCompress = nspf.size - ncaHeaderSize
blocksToCompress = bytesToCompress // blockSize + (
bytesToCompress % blockSize > 0)
compressedblockSizeList = [0] * blocksToCompress
header = b'NCZBLOCK' #Magic
header += b'\x02' #Version
header += b'\x01' #Type
header += b'\x00' #Unused
header += blockSizeExponent.to_bytes(
1, 'little') #blockSizeExponent in bits: 2^x
header += blocksToCompress.to_bytes(
4, 'little') #Amount of Blocks
header += bytesToCompress.to_bytes(
8, 'little') #Decompressed Size
header += b'\x00' * (blocksToCompress * 4)
f.write(header)
decompressedBytes = ncaHeaderSize
compressedBytes = f.tell()
BAR_FMT = u'{desc}{desc_pad}{percentage:3.0f}%|{bar}| {count:{len_total}d}/{total:d} {unit} [{elapsed}<{eta}, {rate:.2f}{unit_pad}{unit}/s]'
bar = enlighten.Counter(total=nspf.size // 1048576,
desc='Compressing',
unit='MiB',
color='cyan',
bar_format=BAR_FMT)
subBars = bar.add_subcounter('green', all_fields=True)
partitions = [
nspf.partition(offset=section.offset,
size=section.size,
n=None,
cryptoType=section.cryptoType,
cryptoKey=section.cryptoKey,
cryptoCounter=bytearray(
section.cryptoCounter),
autoOpen=True) for section in sections
]
partNr = 0
bar.count = nspf.tell() // 1048576
subBars.count = f.tell() // 1048576
bar.refresh()
while True:
buffer = partitions[partNr].read(blockSize)
while (len(buffer) < blockSize
and partNr < len(partitions) - 1):
partitions[partNr].close()
partitions[partNr] = None
partNr += 1
buffer += partitions[partNr].read(blockSize -
len(buffer))
if chunkRelativeBlockID >= TasksPerChunk or len(
buffer) == 0:
while readyForWork.value() < threads:
sleep(0.02)
for i in range(
min(TasksPerChunk,
blocksToCompress - startChunkBlockID)):
lenResult = len(results[i])
compressedBytes += lenResult
compressedblockSizeList[startChunkBlockID +
i] = lenResult
f.write(results[i])
results[i] = b""
if len(buffer) == 0:
sleep(0.02)
pleaseKillYourself.increment()
for i in range(readyForWork.value()):
work.put(None)
while readyForWork.value() > 0:
sleep(0.02)
break
chunkRelativeBlockID = 0
startChunkBlockID = blockID
work.put([
buffer, compressionLevel, compressedblockSizeList,
chunkRelativeBlockID
])
blockID += 1
chunkRelativeBlockID += 1
decompressedBytes += len(buffer)
bar.count = decompressedBytes // 1048576
subBars.count = compressedBytes // 1048576
bar.refresh()
partitions[partNr].close()
partitions[partNr] = None
endPos = f.tell()
bar.count = decompressedBytes // 1048576
subBars.count = compressedBytes // 1048576
bar.close()
written = endPos - startPos
f.seek(blocksHeaderFilePos + 24)
header = b""
for compressedblockSize in compressedblockSizeList:
header += compressedblockSize.to_bytes(4, 'little')
f.write(header)
f.seek(endPos) #Seek to end of file.
Print.info('compressed %d%% %d -> %d - %s' %
(int(written * 100 / nspf.size), decompressedBytes,
written, nspf._path))
writeContainer.resize(newFileName, written)
continue
else:
Print.info('not packed!')
f = writeContainer.add(nspf._path, nspf.size)
nspf.seek(0)
while not nspf.eof():
buffer = nspf.read(CHUNK_SZ)
f.write(buffer)