def supertrim_xci(filepath,
buffer=65536,
outfile=None,
keepupd=False,
level=17,
threads=0,
pos=False,
nthreads=False):
isthreaded = False
if pos != False:
isthreaded = True
elif str(pos) == '0':
isthreaded = True
else:
pos = 0
pos = int(pos)
if isthreaded == False:
try:
exchangefile.deletefile()
except:
pass
f = squirrelXCI(filepath)
t = tqdm(total=0, unit='B', unit_scale=True, leave=False, position=0)
for nspF in f.hfs0:
if str(nspF._path) == "secure":
for ticket in nspF:
if str(ticket._path).endswith('.tik'):
if isthreaded == False:
t.write('- Titlerights: ' + ticket.rightsId)
tk = (str(hex(ticket.getTitleKeyBlock()))[2:]).upper()
if isthreaded == False:
if len(tk) == 30:
tk = '00' + str(tk).upper()
if len(tk) == 31:
tk = '0' + str(tk).upper()
t.write('- Titlekey: ' + tk)
exchangefile.add(ticket.rightsId, tk)
f.flush()
f.close()
t.close()
files_list = sq_tools.ret_xci_offsets(filepath)
files = list()
filesizes = list()
if isthreaded == True and nthreads != False:
tqlist = list()
for i in range(nthreads):
tq = tqdm(total=0,
unit='B',
unit_scale=True,
leave=False,
position=i)
tqlist.append(tq)
fplist = list()
for k in range(len(files_list)):
entry = files_list[k]
fplist.append(entry[0])
for i in range(len(files_list)):
entry = files_list[i]
cnmtfile = entry[0]
if cnmtfile.endswith('.cnmt.nca'):
f = squirrelXCI(filepath)
titleid, titleversion, base_ID, keygeneration, rightsId, RSV, RGV, ctype, metasdkversion, exesdkversion, hasHtmlManual, Installedsize, DeltaSize, ncadata = f.get_data_from_cnmt(
cnmtfile)
f.flush()
f.close()
for j in range(len(ncadata)):
row = ncadata[j]
# print(row)
if row['NCAtype'] != 'Meta':
test1 = str(row['NcaId']) + '.nca'
test2 = str(row['NcaId']) + '.ncz'
if test1 in fplist or test2 in fplist:
# print(str(row['NcaId'])+'.nca')
files.append(str(row['NcaId']) + '.nca')
filesizes.append(int(row['Size']))
else:
# print(str(row['NcaId'])+'.cnmt.nca')
files.append(str(row['NcaId']) + '.cnmt.nca')
filesizes.append(int(row['Size']))
for k in range(len(files_list)):
entry = files_list[k]
fp = entry[0]
sz = int(entry[3])
if fp.endswith('xml'):
files.append(fp)
filesizes.append(sz)
for k in range(len(files_list)):
entry = files_list[k]
fp = entry[0]
sz = int(entry[3])
if fp.endswith('.tik'):
files.append(fp)
filesizes.append(sz)
for k in range(len(files_list)):
entry = files_list[k]
fp = entry[0]
sz = int(entry[3])
if fp.endswith('.cert'):
files.append(fp)
filesizes.append(sz)
sec_hashlist = list()
f = squirrelXCI(filepath)
try:
for file in files:
sha, size, gamecard = f.file_hash(file)
# print(sha)
if sha != False:
sec_hashlist.append(sha)
except BaseException as e:
Print.error('Exception: ' + str(e))
f.flush()
f.close()
xci_header, game_info, sig_padding, xci_certificate, root_header, upd_header, norm_header, sec_header, rootSize, upd_multiplier, norm_multiplier, sec_multiplier = sq_tools.get_xciheader(
files, filesizes, sec_hashlist)
compressionLevel = int(level)
CHUNK_SZ = buffer
if outfile == None:
ofile = filepath[0:-1] + 'z'
else:
ofile = outfile
ofile = os.path.abspath(ofile)
outheader = xci_header
outheader += game_info
outheader += sig_padding
outheader += xci_certificate
outheader += root_header
outheader += upd_header
outheader += norm_header
outheader += sec_header
properheadsize = len(outheader)
compressionLevel = int(level)
CHUNK_SZ = buffer
if outfile == None:
nszPath = filepath[0:-1] + 'z'
else:
nszPath = outfile
nszPath = os.path.abspath(nszPath)
tsize = properheadsize
for sz in filesizes:
tsize += sz
if isthreaded == True:
from colorama import Fore
colors = Fore.__dict__
k = 0
l = pos
for col in colors:
if l > len(colors):
l = l - len(colors)
color = colors[col]
if k == (l + 1):
break
else:
k += 1
t = tqdm(total=tsize,
unit='B',
unit_scale=True,
leave=False,
position=pos,
bar_format="{l_bar}%s{bar}%s{r_bar}" % (color, Fore.RESET))
else:
t = tqdm(total=tsize,
unit='B',
unit_scale=True,
leave=False,
position=0)
if isthreaded == False:
t.write('Compressing with level %d and %d threads' %
(compressionLevel, threads))
t.write('\n %s -> %s \n' % (filepath, nszPath))
newNsp = nutFs.Pfs0.Pfs0Stream(nszPath,
headsize=properheadsize,
mode='wb+')
xcicontainer = Xci(filepath)
# f.compressed_supertrim(buffer,outfile,keepupd,level,threads)
files2 = list()
filesizes2 = list()
for file in files:
for nspF in xcicontainer.hfs0:
if str(nspF._path) == "secure":
for nca in nspF:
if nca._path == file:
if isinstance(nca, nutFs.Nca.Nca) and (
nca.header.contentType
== nutFs.Type.Content.PROGRAM
or nca.header.contentType
== nutFs.Type.Content.PUBLIC_DATA):
if isNcaPacked(nca):
cctx = zstandard.ZstdCompressor(
level=compressionLevel, threads=threads)
newFileName = nca._path[0:-1] + 'z'
f = newNsp.add(newFileName, nca.size, t,
isthreaded)
start = f.tell()
nca.seek(0)
data = nca.read(ncaHeaderSize)
f.write(data)
nca.seek(ncaHeaderSize)
written = ncaHeaderSize
compressor = cctx.stream_writer(f)
sections = get_sections(nca)
header = b'NCZSECTN'
header += len(sections).to_bytes(8, 'little')
i = 0
for fs in sections:
i += 1
header += fs.offset.to_bytes(8, 'little')
header += fs.size.to_bytes(8, 'little')
header += fs.cryptoType.to_bytes(
8, 'little')
header += b'\x00' * 8
header += fs.cryptoKey
header += fs.cryptoCounter
f.write(header)
t.update(len(header))
written += len(header)
timestamp = time.time()
decompressedBytes = ncaHeaderSize
totsize = 0
for fs in sections:
totsize += fs.size
for section in sections:
#print('offset: %x\t\tsize: %x\t\ttype: %d\t\tiv%s' % (section.offset, section.size, section.cryptoType, str(hx(section.cryptoCounter))))
o = nca.partition(
offset=section.offset,
size=section.size,
n=None,
cryptoType=section.cryptoType,
cryptoKey=section.cryptoKey,
cryptoCounter=bytearray(
section.cryptoCounter),
autoOpen=True)
while not o.eof():
buffer = o.read(CHUNK_SZ)
t.update(len(buffer))
if len(buffer) == 0:
raise IOError('read failed')
written += compressor.write(buffer)
decompressedBytes += len(buffer)
compressor.flush(zstandard.FLUSH_FRAME)
elapsed = time.time() - timestamp
minutes = elapsed / 60
seconds = elapsed % 60
speed = 0 if elapsed == 0 else (nca.size /
elapsed)
written = f.tell() - start
if isthreaded == False:
t.write(
'\n * Compressed at %d%% from %s to %s - %s'
% (int(written * 100 / nca.size),
str(
sq_tools.getSize(
decompressedBytes)),
str(sq_tools.getSize(written)),
nca._path))
t.write(
' * Compressed in %02d:%02d at speed: %.1f MB/s\n'
%
(minutes, seconds, speed / 1000000.0))
newNsp.resize(newFileName, written)
files2.append(newFileName)
filesizes2.append(written)
continue
else:
if isthreaded == False:
t.write('not packed!')
f = newNsp.add(nca._path, nca.size, t, isthreaded)
files2.append(nca._path)
filesizes2.append(nca.size)
nca.seek(0)
while not nca.eof():
buffer = nca.read(CHUNK_SZ)
t.update(len(buffer))
f.write(buffer)
t.close()
newNsp.close()
xci_header, game_info, sig_padding, xci_certificate, root_header, upd_header, norm_header, sec_header, rootSize, upd_multiplier, norm_multiplier, sec_multiplier = sq_tools.get_xciheader(
files2, filesizes2, sec_hashlist)
outheader = xci_header
outheader += game_info
outheader += sig_padding
outheader += xci_certificate
outheader += root_header
outheader += upd_header
outheader += norm_header
outheader += sec_header
with open(nszPath, 'rb+') as o:
o.seek(0)
o.write(outheader)
try:
exchangefile.deletefile()
except:
pass
if isthreaded == True and nthreads != False:
for i in range(nthreads):
tqlist[i].close()
return nszPath
def gen_multi_file_header(prlist, filelist):
oflist = []
osizelist = []
ototlist = []
files = []
totSize = 0
for i in range(len(prlist)):
for j in prlist[i][4]:
el = j[0]
if el.endswith('.nca'):
oflist.append(j[0])
#print(j[0])
totSize = totSize + j[1]
#print(j[1])
ototlist.append(j[0])
sec_hashlist = list()
GClist = list()
# print(filelist)
for file in oflist:
for filepath in filelist:
if filepath.endswith('.nsp') or filepath.endswith('.nsz'):
try:
f = squirrelNSP(filepath)
sha, size, gamecard = f.file_hash(file)
if sha != False:
sec_hashlist.append(sha)
osizelist.append(size)
GClist.append([file, gamecard])
f.flush()
f.close()
except BaseException as e:
Print.error('Exception: ' + str(e))
if filepath.endswith('.xci') or filepath.endswith('.xcz'):
try:
f = squirrelXCI(filepath)
sha, size, gamecard = f.file_hash(file)
if sha != False:
sec_hashlist.append(sha)
osizelist.append(size)
GClist.append([file, gamecard])
f.flush()
f.close()
except BaseException as e:
Print.error('Exception: ' + str(e))
xci_header, game_info, sig_padding, xci_certificate, root_header, upd_header, norm_header, sec_header, rootSize, upd_multiplier, norm_multiplier, sec_multiplier = sq_tools.get_xciheader(
oflist, osizelist, sec_hashlist)
totSize = len(xci_header) + len(game_info) + len(sig_padding) + len(
xci_certificate) + rootSize
outheader = xci_header
outheader += game_info
outheader += sig_padding
outheader += xci_certificate
outheader += root_header
outheader += upd_header
outheader += norm_header
outheader += sec_header
properheadsize = len(outheader)
return outheader, properheadsize, totSize, oflist
def get_header_size(filepath):
properheadsize=0;sz=0
if filepath.endswith('xci') or filepath.endswith('xcz'):
files_list=sq_tools.ret_xci_offsets(filepath)
files=list();filesizes=list()
fplist=list()
for k in range(len(files_list)):
entry=files_list[k]
fplist.append(entry[0])
for i in range(len(files_list)):
entry=files_list[i]
cnmtfile=entry[0]
if cnmtfile.endswith('.cnmt.nca'):
f=squirrelXCI(filepath)
titleid,titleversion,base_ID,keygeneration,rightsId,RSV,RGV,ctype,metasdkversion,exesdkversion,hasHtmlManual,Installedsize,DeltaSize,ncadata=f.get_data_from_cnmt(cnmtfile)
for j in range(len(ncadata)):
row=ncadata[j]
# print(row)
if row['NCAtype']!='Meta':
test1=str(row['NcaId'])+'.nca';test2=str(row['NcaId'])+'.ncz'
if test1 in fplist or test2 in fplist:
# print(str(row['NcaId'])+'.nca')
files.append(str(row['NcaId'])+'.nca')
filesizes.append(int(row['Size']))
sz+=int(row['Size'])
elif row['NCAtype']=='Meta':
# print(str(row['NcaId'])+'.cnmt.nca')
files.append(str(row['NcaId'])+'.cnmt.nca')
filesizes.append(int(row['Size']))
sz+=int(row['Size'])
sec_hashlist=list()
try:
for file in files:
sha,size,gamecard=f.file_hash(file)
# print(sha)
if sha != False:
sec_hashlist.append(sha)
except BaseException as e:
Print.error('Exception: ' + str(e))
f.flush()
f.close()
xci_header,game_info,sig_padding,xci_certificate,root_header,upd_header,norm_header,sec_header,rootSize,upd_multiplier,norm_multiplier,sec_multiplier=sq_tools.get_xciheader(files,filesizes,sec_hashlist)
outheader=xci_header
outheader+=game_info
outheader+=sig_padding
outheader+=xci_certificate
outheader+=root_header
outheader+=upd_header
outheader+=norm_header
outheader+=sec_header
elif filepath.endswith('nsp') or filepath.endswith('nsz'):
files_list=sq_tools.ret_nsp_offsets(filepath)
files=list();filesizes=list()
fplist=list()
for k in range(len(files_list)):
entry=files_list[k]
fplist.append(entry[0])
for i in range(len(files_list)):
entry=files_list[i]
cnmtfile=entry[0]
if cnmtfile.endswith('.cnmt.nca'):
f=squirrelNSP(filepath)
titleid,titleversion,base_ID,keygeneration,rightsId,RSV,RGV,ctype,metasdkversion,exesdkversion,hasHtmlManual,Installedsize,DeltaSize,ncadata=f.get_data_from_cnmt(cnmtfile)
f.flush()
f.close()
for j in range(len(ncadata)):
row=ncadata[j]
# print(row)
if row['NCAtype']!='Meta':
test1=str(row['NcaId'])+'.nca';test2=str(row['NcaId'])+'.ncz'
if test1 in fplist or test2 in fplist:
# print(str(row['NcaId'])+'.nca')
files.append(str(row['NcaId'])+'.nca')
filesizes.append(int(row['Size']))
sz+=int(row['Size'])
elif row['NCAtype']=='Meta':
# print(str(row['NcaId'])+'.cnmt.nca')
files.append(str(row['NcaId'])+'.cnmt.nca')
filesizes.append(int(row['Size']))
sz+=int(row['Size'])
f.flush()
f.close()
outheader = sq_tools.gen_nsp_header(files,filesizes)
properheadsize=len(outheader)
return properheadsize,keygeneration,sz
def decompress_xcz(remote, ofolder):
buf = 64 * 1024
buffer = buf
endname = remote.name[:-1] + 'i'
output = os.path.join(ofolder, endname)
files_list = DriveTools.get_files_from_head(remote, remote.name)
remote.rewind()
# print(files_list)
print('Decompressing {}'.format(remote.name))
print('- Parsing headers...')
files = list()
filesizes = list()
fplist = list()
for k in range(len(files_list)):
entry = files_list[k]
fplist.append(entry[0])
for i in range(len(files_list)):
entry = files_list[i]
cnmtfile = entry[0]
metadict = {}
if cnmtfile.endswith('.cnmt.nca'):
metadict, d1, d2 = DriveTools.get_cnmt_data(target=cnmtfile,
file=remote)
ncadata = metadict['ncadata']
for j in range(len(ncadata)):
row = ncadata[j]
# print(row)
if row['NCAtype'] != 'Meta':
test1 = str(row['NcaId']) + '.nca'
test2 = str(row['NcaId']) + '.ncz'
if test1 in fplist or test2 in fplist:
# print(str(row['NcaId'])+'.nca')
files.append(str(row['NcaId']) + '.nca')
filesizes.append(int(row['Size']))
else:
# print(str(row['NcaId'])+'.cnmt.nca')
files.append(str(row['NcaId']) + '.cnmt.nca')
filesizes.append(int(row['Size']))
for k in range(len(files_list)):
entry = files_list[k]
fp = entry[0]
sz = int(entry[3])
if fp.endswith('xml'):
files.append(fp)
filesizes.append(sz)
for k in range(len(files_list)):
entry = files_list[k]
fp = entry[0]
sz = int(entry[3])
if fp.endswith('.tik'):
files.append(fp)
filesizes.append(sz)
for k in range(len(files_list)):
entry = files_list[k]
fp = entry[0]
sz = int(entry[3])
if fp.endswith('.cert'):
files.append(fp)
filesizes.append(sz)
sec_hashlist = list()
try:
for target in files:
sha, size, gamecard = DriveTools.file_hash(remote, target,
files_list)
# print(sha)
if sha != False:
sec_hashlist.append(sha)
except BaseException as e:
Print.error('Exception: ' + str(e))
# print(sec_hashlist)
xci_header, game_info, sig_padding, xci_certificate, root_header, upd_header, norm_header, sec_header, rootSize, upd_multiplier, norm_multiplier, sec_multiplier = sq_tools.get_xciheader(
files, filesizes, sec_hashlist)
outheader = xci_header
outheader += game_info
outheader += sig_padding
outheader += xci_certificate
outheader += root_header
outheader += upd_header
outheader += norm_header
outheader += sec_header
properheadsize = len(outheader)
totsize = properheadsize
for s in filesizes:
totsize += s
# Hex.dump(outheader)
# print(totsize)
print(files)
t = tqdm(total=totsize, unit='B', unit_scale=True, leave=False)
with open(output, 'wb') as o:
o.write(outheader)
t.update(len(outheader))
for file in files:
if file.endswith('cnmt.nca'):
for i in range(len(files_list)):
if str(files_list[i][0]).lower() == str(file).lower():
nca_name = files_list[i][0]
off1 = files_list[i][1]
off2 = files_list[i][2]
nca_size = files_list[i][3]
break
t.write('- Appending {}'.format(nca_name))
data = remote.read_at(off1, nca_size)
with open(output, 'ab') as o:
o.write(data)
t.update(len(data))
elif file.endswith('.nca'):
for i in range(len(files_list)):
if str(files_list[i][0]).lower() == str(file).lower():
nca_name = files_list[i][0]
off1 = files_list[i][1]
off2 = files_list[i][2]
nca_size = files_list[i][3]
t.write('- Appending {}'.format(nca_name))
o = open(output, 'ab+')
remote.seek(off1, off2)
for data in remote.response.iter_content(chunk_size=buf):
o.write(data)
t.update(len(data))
o.flush()
if not data:
o.close()
break
elif (str(files_list[i][0]).lower())[:-1] == (
str(file).lower())[:-1] and str(
files_list[i][0]).endswith('ncz'):
ncz_name = files_list[i][0]
off1 = files_list[i][1]
off2 = files_list[i][2]
sz = files_list[i][3]
t.write('- Calculating compressed sections for: ' +
ncz_name)
remote.seek(off1, off2)
header = remote.read(0x4000)
magic = readInt64(remote)
sectionCount = readInt64(remote)
sections = []
for i in range(sectionCount):
sections.append(Section(remote))
t.write(' Detected {} sections'.format(str(sectionCount)))
with open(output, 'rb+') as o:
o.seek(0, os.SEEK_END)
curr_off = o.tell()
t.write('- Appending decompressed {}'.format(ncz_name))
t.write(' Writing nca header')
o.write(header)
t.update(len(header))
timestamp = time.time()
t.write(' Writing decompressed body in plaintext')
count = 0
checkstarter = 0
dctx = zstandard.ZstdDecompressor()
hd = Private.get_html_header(remote.access_token,
remote.position, off2)
remote.get_session(hd)
reader = dctx.stream_reader(remote.response.raw,
read_size=buffer)
c = 0
spsize = 0
for s in sections:
end = s.offset + s.size
if s.cryptoType == 1: #plain text
t.write(' * Section {} is plaintext'.format(
str(c)))
t.write(' %x - %d bytes, Crypto type %d' %
((s.offset), s.size, s.cryptoType))
spsize += s.size
end = s.offset + s.size
i = s.offset
while i < end:
chunkSz = buffer if end - i > buffer else end - i
chunk = reader.read(chunkSz)
if not len(chunk):
break
o.write(chunk)
t.update(len(chunk))
i += chunkSz
elif s.cryptoType not in (3, 4):
raise IOError('Unknown crypto type: %d' %
s.cryptoType)
else:
t.write(' * Section {} needs decompression'.
format(str(c)))
t.write(' %x - %d bytes, Crypto type %d' %
((s.offset), s.size, s.cryptoType))
t.write(' Key: %s, IV: %s' %
(str(hx(s.cryptoKey)),
str(hx(s.cryptoCounter))))
crypto = AESCTR(s.cryptoKey, s.cryptoCounter)
spsize += s.size
test = int(spsize / (buffer))
i = s.offset
while i < end:
crypto.seek(i)
chunkSz = buffer if end - i > buffer else end - i
chunk = reader.read(chunkSz)
if not len(chunk):
break
o.write(crypto.encrypt(chunk))
t.update(len(chunk))
i += chunkSz
c += 1
elapsed = time.time() - timestamp
minutes = elapsed / 60
seconds = elapsed % 60
speed = 0 if elapsed == 0 else (spsize / elapsed)
t.write(
'\n Decompressed in %02d:%02d at speed: %.1f MB/s\n'
% (minutes, seconds, speed / 1000000.0))
else:
for i in range(len(files_list)):
if str(files_list[i][0]).lower() == str(file).lower():
file_name = files_list[i][0]
off1 = files_list[i][1]
off2 = files_list[i][2]
file_size = files_list[i][3]
break
t.write('- Appending {}'.format(file_name))
data = remote.read_at(off1, file_size)
with open(output, 'ab') as o:
o.write(data)
t.update(len(data))
t.close()
