def supertrim_xci(filepath, buffer=65536, outfile=None, keepupd=False, level=17, threads=0, pos=False, nthreads=False): isthreaded = False if pos != False: isthreaded = True elif str(pos) == '0': isthreaded = True else: pos = 0 pos = int(pos) if isthreaded == False: try: exchangefile.deletefile() except: pass f = squirrelXCI(filepath) t = tqdm(total=0, unit='B', unit_scale=True, leave=False, position=0) for nspF in f.hfs0: if str(nspF._path) == "secure": for ticket in nspF: if str(ticket._path).endswith('.tik'): if isthreaded == False: t.write('- Titlerights: ' + ticket.rightsId) tk = (str(hex(ticket.getTitleKeyBlock()))[2:]).upper() if isthreaded == False: if len(tk) == 30: tk = '00' + str(tk).upper() if len(tk) == 31: tk = '0' + str(tk).upper() t.write('- Titlekey: ' + tk) exchangefile.add(ticket.rightsId, tk) f.flush() f.close() t.close() files_list = sq_tools.ret_xci_offsets(filepath) files = list() filesizes = list() if isthreaded == True and nthreads != False: tqlist = list() for i in range(nthreads): tq = tqdm(total=0, unit='B', unit_scale=True, leave=False, position=i) tqlist.append(tq) fplist = list() for k in range(len(files_list)): entry = files_list[k] fplist.append(entry[0]) for i in range(len(files_list)): entry = files_list[i] cnmtfile = entry[0] if cnmtfile.endswith('.cnmt.nca'): f = squirrelXCI(filepath) titleid, titleversion, base_ID, keygeneration, rightsId, RSV, RGV, ctype, metasdkversion, exesdkversion, hasHtmlManual, Installedsize, DeltaSize, ncadata = f.get_data_from_cnmt( cnmtfile) f.flush() f.close() for j in range(len(ncadata)): row = ncadata[j] # print(row) if row['NCAtype'] != 'Meta': test1 = str(row['NcaId']) + '.nca' test2 = str(row['NcaId']) + '.ncz' if test1 in fplist or test2 in fplist: # print(str(row['NcaId'])+'.nca') files.append(str(row['NcaId']) + '.nca') filesizes.append(int(row['Size'])) else: # print(str(row['NcaId'])+'.cnmt.nca') files.append(str(row['NcaId']) + '.cnmt.nca') filesizes.append(int(row['Size'])) for k in range(len(files_list)): entry = files_list[k] fp = entry[0] sz = int(entry[3]) if fp.endswith('xml'): files.append(fp) filesizes.append(sz) for k in range(len(files_list)): entry = files_list[k] fp = entry[0] sz = int(entry[3]) if fp.endswith('.tik'): files.append(fp) filesizes.append(sz) for k in range(len(files_list)): entry = files_list[k] fp = entry[0] sz = int(entry[3]) if fp.endswith('.cert'): files.append(fp) filesizes.append(sz) sec_hashlist = list() f = squirrelXCI(filepath) try: for file in files: sha, size, gamecard = f.file_hash(file) # print(sha) if sha != False: sec_hashlist.append(sha) except BaseException as e: Print.error('Exception: ' + str(e)) f.flush() f.close() xci_header, game_info, sig_padding, xci_certificate, root_header, upd_header, norm_header, sec_header, rootSize, upd_multiplier, norm_multiplier, sec_multiplier = sq_tools.get_xciheader( files, filesizes, sec_hashlist) compressionLevel = int(level) CHUNK_SZ = buffer if outfile == None: ofile = filepath[0:-1] + 'z' else: ofile = outfile ofile = os.path.abspath(ofile) outheader = xci_header outheader += game_info outheader += sig_padding outheader += xci_certificate outheader += root_header outheader += upd_header outheader += norm_header outheader += sec_header properheadsize = len(outheader) compressionLevel = int(level) CHUNK_SZ = buffer if outfile == None: nszPath = filepath[0:-1] + 'z' else: nszPath = outfile nszPath = os.path.abspath(nszPath) tsize = properheadsize for sz in filesizes: tsize += sz if isthreaded == True: from colorama import Fore colors = Fore.__dict__ k = 0 l = pos for col in colors: if l > len(colors): l = l - len(colors) color = colors[col] if k == (l + 1): break else: k += 1 t = tqdm(total=tsize, unit='B', unit_scale=True, leave=False, position=pos, bar_format="{l_bar}%s{bar}%s{r_bar}" % (color, Fore.RESET)) else: t = tqdm(total=tsize, unit='B', unit_scale=True, leave=False, position=0) if isthreaded == False: t.write('Compressing with level %d and %d threads' % (compressionLevel, threads)) t.write('\n %s -> %s \n' % (filepath, nszPath)) newNsp = nutFs.Pfs0.Pfs0Stream(nszPath, headsize=properheadsize, mode='wb+') xcicontainer = Xci(filepath) # f.compressed_supertrim(buffer,outfile,keepupd,level,threads) files2 = list() filesizes2 = list() for file in files: for nspF in xcicontainer.hfs0: if str(nspF._path) == "secure": for nca in nspF: if nca._path == file: if isinstance(nca, nutFs.Nca.Nca) and ( nca.header.contentType == nutFs.Type.Content.PROGRAM or nca.header.contentType == nutFs.Type.Content.PUBLIC_DATA): if isNcaPacked(nca): cctx = zstandard.ZstdCompressor( level=compressionLevel, threads=threads) newFileName = nca._path[0:-1] + 'z' f = newNsp.add(newFileName, nca.size, t, isthreaded) start = f.tell() nca.seek(0) data = nca.read(ncaHeaderSize) f.write(data) nca.seek(ncaHeaderSize) written = ncaHeaderSize compressor = cctx.stream_writer(f) sections = get_sections(nca) header = b'NCZSECTN' header += len(sections).to_bytes(8, 'little') i = 0 for fs in sections: i += 1 header += fs.offset.to_bytes(8, 'little') header += fs.size.to_bytes(8, 'little') header += fs.cryptoType.to_bytes( 8, 'little') header += b'\x00' * 8 header += fs.cryptoKey header += fs.cryptoCounter f.write(header) t.update(len(header)) written += len(header) timestamp = time.time() decompressedBytes = ncaHeaderSize totsize = 0 for fs in sections: totsize += fs.size for section in sections: #print('offset: %x\t\tsize: %x\t\ttype: %d\t\tiv%s' % (section.offset, section.size, section.cryptoType, str(hx(section.cryptoCounter)))) o = nca.partition( offset=section.offset, size=section.size, n=None, cryptoType=section.cryptoType, cryptoKey=section.cryptoKey, cryptoCounter=bytearray( section.cryptoCounter), autoOpen=True) while not o.eof(): buffer = o.read(CHUNK_SZ) t.update(len(buffer)) if len(buffer) == 0: raise IOError('read failed') written += compressor.write(buffer) decompressedBytes += len(buffer) compressor.flush(zstandard.FLUSH_FRAME) elapsed = time.time() - timestamp minutes = elapsed / 60 seconds = elapsed % 60 speed = 0 if elapsed == 0 else (nca.size / elapsed) written = f.tell() - start if isthreaded == False: t.write( '\n * Compressed at %d%% from %s to %s - %s' % (int(written * 100 / nca.size), str( sq_tools.getSize( decompressedBytes)), str(sq_tools.getSize(written)), nca._path)) t.write( ' * Compressed in %02d:%02d at speed: %.1f MB/s\n' % (minutes, seconds, speed / 1000000.0)) newNsp.resize(newFileName, written) files2.append(newFileName) filesizes2.append(written) continue else: if isthreaded == False: t.write('not packed!') f = newNsp.add(nca._path, nca.size, t, isthreaded) files2.append(nca._path) filesizes2.append(nca.size) nca.seek(0) while not nca.eof(): buffer = nca.read(CHUNK_SZ) t.update(len(buffer)) f.write(buffer) t.close() newNsp.close() xci_header, game_info, sig_padding, xci_certificate, root_header, upd_header, norm_header, sec_header, rootSize, upd_multiplier, norm_multiplier, sec_multiplier = sq_tools.get_xciheader( files2, filesizes2, sec_hashlist) outheader = xci_header outheader += game_info outheader += sig_padding outheader += xci_certificate outheader += root_header outheader += upd_header outheader += norm_header outheader += sec_header with open(nszPath, 'rb+') as o: o.seek(0) o.write(outheader) try: exchangefile.deletefile() except: pass if isthreaded == True and nthreads != False: for i in range(nthreads): tqlist[i].close() return nszPath
def gen_multi_file_header(prlist, filelist): oflist = [] osizelist = [] ototlist = [] files = [] totSize = 0 for i in range(len(prlist)): for j in prlist[i][4]: el = j[0] if el.endswith('.nca'): oflist.append(j[0]) #print(j[0]) totSize = totSize + j[1] #print(j[1]) ototlist.append(j[0]) sec_hashlist = list() GClist = list() # print(filelist) for file in oflist: for filepath in filelist: if filepath.endswith('.nsp') or filepath.endswith('.nsz'): try: f = squirrelNSP(filepath) sha, size, gamecard = f.file_hash(file) if sha != False: sec_hashlist.append(sha) osizelist.append(size) GClist.append([file, gamecard]) f.flush() f.close() except BaseException as e: Print.error('Exception: ' + str(e)) if filepath.endswith('.xci') or filepath.endswith('.xcz'): try: f = squirrelXCI(filepath) sha, size, gamecard = f.file_hash(file) if sha != False: sec_hashlist.append(sha) osizelist.append(size) GClist.append([file, gamecard]) f.flush() f.close() except BaseException as e: Print.error('Exception: ' + str(e)) xci_header, game_info, sig_padding, xci_certificate, root_header, upd_header, norm_header, sec_header, rootSize, upd_multiplier, norm_multiplier, sec_multiplier = sq_tools.get_xciheader( oflist, osizelist, sec_hashlist) totSize = len(xci_header) + len(game_info) + len(sig_padding) + len( xci_certificate) + rootSize outheader = xci_header outheader += game_info outheader += sig_padding outheader += xci_certificate outheader += root_header outheader += upd_header outheader += norm_header outheader += sec_header properheadsize = len(outheader) return outheader, properheadsize, totSize, oflist
def get_header_size(filepath): properheadsize=0;sz=0 if filepath.endswith('xci') or filepath.endswith('xcz'): files_list=sq_tools.ret_xci_offsets(filepath) files=list();filesizes=list() fplist=list() for k in range(len(files_list)): entry=files_list[k] fplist.append(entry[0]) for i in range(len(files_list)): entry=files_list[i] cnmtfile=entry[0] if cnmtfile.endswith('.cnmt.nca'): f=squirrelXCI(filepath) titleid,titleversion,base_ID,keygeneration,rightsId,RSV,RGV,ctype,metasdkversion,exesdkversion,hasHtmlManual,Installedsize,DeltaSize,ncadata=f.get_data_from_cnmt(cnmtfile) for j in range(len(ncadata)): row=ncadata[j] # print(row) if row['NCAtype']!='Meta': test1=str(row['NcaId'])+'.nca';test2=str(row['NcaId'])+'.ncz' if test1 in fplist or test2 in fplist: # print(str(row['NcaId'])+'.nca') files.append(str(row['NcaId'])+'.nca') filesizes.append(int(row['Size'])) sz+=int(row['Size']) elif row['NCAtype']=='Meta': # print(str(row['NcaId'])+'.cnmt.nca') files.append(str(row['NcaId'])+'.cnmt.nca') filesizes.append(int(row['Size'])) sz+=int(row['Size']) sec_hashlist=list() try: for file in files: sha,size,gamecard=f.file_hash(file) # print(sha) if sha != False: sec_hashlist.append(sha) except BaseException as e: Print.error('Exception: ' + str(e)) f.flush() f.close() xci_header,game_info,sig_padding,xci_certificate,root_header,upd_header,norm_header,sec_header,rootSize,upd_multiplier,norm_multiplier,sec_multiplier=sq_tools.get_xciheader(files,filesizes,sec_hashlist) outheader=xci_header outheader+=game_info outheader+=sig_padding outheader+=xci_certificate outheader+=root_header outheader+=upd_header outheader+=norm_header outheader+=sec_header elif filepath.endswith('nsp') or filepath.endswith('nsz'): files_list=sq_tools.ret_nsp_offsets(filepath) files=list();filesizes=list() fplist=list() for k in range(len(files_list)): entry=files_list[k] fplist.append(entry[0]) for i in range(len(files_list)): entry=files_list[i] cnmtfile=entry[0] if cnmtfile.endswith('.cnmt.nca'): f=squirrelNSP(filepath) titleid,titleversion,base_ID,keygeneration,rightsId,RSV,RGV,ctype,metasdkversion,exesdkversion,hasHtmlManual,Installedsize,DeltaSize,ncadata=f.get_data_from_cnmt(cnmtfile) f.flush() f.close() for j in range(len(ncadata)): row=ncadata[j] # print(row) if row['NCAtype']!='Meta': test1=str(row['NcaId'])+'.nca';test2=str(row['NcaId'])+'.ncz' if test1 in fplist or test2 in fplist: # print(str(row['NcaId'])+'.nca') files.append(str(row['NcaId'])+'.nca') filesizes.append(int(row['Size'])) sz+=int(row['Size']) elif row['NCAtype']=='Meta': # print(str(row['NcaId'])+'.cnmt.nca') files.append(str(row['NcaId'])+'.cnmt.nca') filesizes.append(int(row['Size'])) sz+=int(row['Size']) f.flush() f.close() outheader = sq_tools.gen_nsp_header(files,filesizes) properheadsize=len(outheader) return properheadsize,keygeneration,sz
def decompress_xcz(remote, ofolder): buf = 64 * 1024 buffer = buf endname = remote.name[:-1] + 'i' output = os.path.join(ofolder, endname) files_list = DriveTools.get_files_from_head(remote, remote.name) remote.rewind() # print(files_list) print('Decompressing {}'.format(remote.name)) print('- Parsing headers...') files = list() filesizes = list() fplist = list() for k in range(len(files_list)): entry = files_list[k] fplist.append(entry[0]) for i in range(len(files_list)): entry = files_list[i] cnmtfile = entry[0] metadict = {} if cnmtfile.endswith('.cnmt.nca'): metadict, d1, d2 = DriveTools.get_cnmt_data(target=cnmtfile, file=remote) ncadata = metadict['ncadata'] for j in range(len(ncadata)): row = ncadata[j] # print(row) if row['NCAtype'] != 'Meta': test1 = str(row['NcaId']) + '.nca' test2 = str(row['NcaId']) + '.ncz' if test1 in fplist or test2 in fplist: # print(str(row['NcaId'])+'.nca') files.append(str(row['NcaId']) + '.nca') filesizes.append(int(row['Size'])) else: # print(str(row['NcaId'])+'.cnmt.nca') files.append(str(row['NcaId']) + '.cnmt.nca') filesizes.append(int(row['Size'])) for k in range(len(files_list)): entry = files_list[k] fp = entry[0] sz = int(entry[3]) if fp.endswith('xml'): files.append(fp) filesizes.append(sz) for k in range(len(files_list)): entry = files_list[k] fp = entry[0] sz = int(entry[3]) if fp.endswith('.tik'): files.append(fp) filesizes.append(sz) for k in range(len(files_list)): entry = files_list[k] fp = entry[0] sz = int(entry[3]) if fp.endswith('.cert'): files.append(fp) filesizes.append(sz) sec_hashlist = list() try: for target in files: sha, size, gamecard = DriveTools.file_hash(remote, target, files_list) # print(sha) if sha != False: sec_hashlist.append(sha) except BaseException as e: Print.error('Exception: ' + str(e)) # print(sec_hashlist) xci_header, game_info, sig_padding, xci_certificate, root_header, upd_header, norm_header, sec_header, rootSize, upd_multiplier, norm_multiplier, sec_multiplier = sq_tools.get_xciheader( files, filesizes, sec_hashlist) outheader = xci_header outheader += game_info outheader += sig_padding outheader += xci_certificate outheader += root_header outheader += upd_header outheader += norm_header outheader += sec_header properheadsize = len(outheader) totsize = properheadsize for s in filesizes: totsize += s # Hex.dump(outheader) # print(totsize) print(files) t = tqdm(total=totsize, unit='B', unit_scale=True, leave=False) with open(output, 'wb') as o: o.write(outheader) t.update(len(outheader)) for file in files: if file.endswith('cnmt.nca'): for i in range(len(files_list)): if str(files_list[i][0]).lower() == str(file).lower(): nca_name = files_list[i][0] off1 = files_list[i][1] off2 = files_list[i][2] nca_size = files_list[i][3] break t.write('- Appending {}'.format(nca_name)) data = remote.read_at(off1, nca_size) with open(output, 'ab') as o: o.write(data) t.update(len(data)) elif file.endswith('.nca'): for i in range(len(files_list)): if str(files_list[i][0]).lower() == str(file).lower(): nca_name = files_list[i][0] off1 = files_list[i][1] off2 = files_list[i][2] nca_size = files_list[i][3] t.write('- Appending {}'.format(nca_name)) o = open(output, 'ab+') remote.seek(off1, off2) for data in remote.response.iter_content(chunk_size=buf): o.write(data) t.update(len(data)) o.flush() if not data: o.close() break elif (str(files_list[i][0]).lower())[:-1] == ( str(file).lower())[:-1] and str( files_list[i][0]).endswith('ncz'): ncz_name = files_list[i][0] off1 = files_list[i][1] off2 = files_list[i][2] sz = files_list[i][3] t.write('- Calculating compressed sections for: ' + ncz_name) remote.seek(off1, off2) header = remote.read(0x4000) magic = readInt64(remote) sectionCount = readInt64(remote) sections = [] for i in range(sectionCount): sections.append(Section(remote)) t.write(' Detected {} sections'.format(str(sectionCount))) with open(output, 'rb+') as o: o.seek(0, os.SEEK_END) curr_off = o.tell() t.write('- Appending decompressed {}'.format(ncz_name)) t.write(' Writing nca header') o.write(header) t.update(len(header)) timestamp = time.time() t.write(' Writing decompressed body in plaintext') count = 0 checkstarter = 0 dctx = zstandard.ZstdDecompressor() hd = Private.get_html_header(remote.access_token, remote.position, off2) remote.get_session(hd) reader = dctx.stream_reader(remote.response.raw, read_size=buffer) c = 0 spsize = 0 for s in sections: end = s.offset + s.size if s.cryptoType == 1: #plain text t.write(' * Section {} is plaintext'.format( str(c))) t.write(' %x - %d bytes, Crypto type %d' % ((s.offset), s.size, s.cryptoType)) spsize += s.size end = s.offset + s.size i = s.offset while i < end: chunkSz = buffer if end - i > buffer else end - i chunk = reader.read(chunkSz) if not len(chunk): break o.write(chunk) t.update(len(chunk)) i += chunkSz elif s.cryptoType not in (3, 4): raise IOError('Unknown crypto type: %d' % s.cryptoType) else: t.write(' * Section {} needs decompression'. format(str(c))) t.write(' %x - %d bytes, Crypto type %d' % ((s.offset), s.size, s.cryptoType)) t.write(' Key: %s, IV: %s' % (str(hx(s.cryptoKey)), str(hx(s.cryptoCounter)))) crypto = AESCTR(s.cryptoKey, s.cryptoCounter) spsize += s.size test = int(spsize / (buffer)) i = s.offset while i < end: crypto.seek(i) chunkSz = buffer if end - i > buffer else end - i chunk = reader.read(chunkSz) if not len(chunk): break o.write(crypto.encrypt(chunk)) t.update(len(chunk)) i += chunkSz c += 1 elapsed = time.time() - timestamp minutes = elapsed / 60 seconds = elapsed % 60 speed = 0 if elapsed == 0 else (spsize / elapsed) t.write( '\n Decompressed in %02d:%02d at speed: %.1f MB/s\n' % (minutes, seconds, speed / 1000000.0)) else: for i in range(len(files_list)): if str(files_list[i][0]).lower() == str(file).lower(): file_name = files_list[i][0] off1 = files_list[i][1] off2 = files_list[i][2] file_size = files_list[i][3] break t.write('- Appending {}'.format(file_name)) data = remote.read_at(off1, file_size) with open(output, 'ab') as o: o.write(data) t.update(len(data)) t.close()