def downloadThread(i): if not hasCdn: return Print.info('starting thread ' + str(i)) global status while Config.isRunning and not Titles.queue.empty(): try: id = Titles.queue.shift() if id and Titles.contains(id): activeDownloads[i] = 1 t = Titles.get(id) path = cdn.downloadTitle(t.id.lower(), None, t.key) if path and os.path.isfile(path): nsp = Fs.Nsp(path, None) nsp.move() Nsps.save() if status is not None: status.add() activeDownloads[i] = 0 else: time.sleep(1) except KeyboardInterrupt: pass except BaseException as e: Print.error('downloadThread exception: ' + str(e)) traceback.print_exc(file=sys.stdout) activeDownloads[i] = 0 Print.info('ending thread ' + str(i))
def updateVersions(force=True): initTitles() initFiles() i = 0 for k, t in Titles.items(): if force or t.version is None: if (t.isDLC or t.isUpdate or Config.download.base) and (not t.isDLC or Config.download.DLC) and (not t.isDemo or Config.download.demo) and (not t.isUpdate or Config.download.update) and ( t.key or Config.download.sansTitleKey) and (len(Config.titleWhitelist) == 0 or t.id in Config.titleWhitelist) and t.id not in Config.titleBlacklist: v = t.lastestVersion(True) Print.info("%s[%s] v = %s" % (str(t.name), str(t.id), str(v))) i = i + 1 if i % 20 == 0: Titles.save() for t in list(Titles.data().values()): if not t.isUpdate and not t.isDLC and t.updateId and t.updateId and not Titles.contains(t.updateId): u = Title.Title() u.setId(t.updateId) if u.lastestVersion(): Titles.set(t.updateId, u) Print.info("%s[%s] FOUND" % (str(t.name), str(u.id))) i = i + 1 if i % 20 == 0: Titles.save() Titles.save()
def downloadThread(i): Print.info('starting thread ' + str(i)) global status while Config.isRunning: try: id = Titles.queue.shift() if id and Titles.contains(id): activeDownloads[i] = 1 t = Titles.get(id) path = CDNSP.download_game(t.id.lower(), t.lastestVersion(), t.key, True, '', True) if os.path.isfile(path): nsp = Fs.Nsp(path, None) nsp.move() Nsps.files[nsp.path] = nsp Nsps.save() status.add() activeDownloads[i] = 0 else: time.sleep(1) except KeyboardInterrupt: pass except BaseException as e: Print.error(str(e)) activeDownloads[i] = 0 Print.info('ending thread ' + str(i))
def getFrontArtBoxImage(request, response): return getTitleImage(request, response) if len(request.bits) < 3: return Server.Response404(request, response) id = request.bits[2] #width = int(request.bits[3]) #if width < 32 or width > 512: # return Server.Response404(request, response) if not Titles.contains(id): return Server.Response404(request, response) path = Titles.get(id).frontBoxArtFile() if not path: return Server.Response404(request, response) response.setMime(path) response.headers['Cache-Control'] = 'max-age=31536000' if os.path.isfile(path): with open(path, 'rb') as f: response.write(f.read()) return Server.Response500(request, response)
def isActive(self, skipKeyCheck=False): if self.id[0:13] == '0100000000000': return False base = self if self.isDLC or self.isUpdate: baseId = getBaseId(self.id) if Titles.contains(baseId): base = Titles.get(baseId) if (self.isDLC or self.isUpdate or Config.download.base) and (not self.isDLC or Config.download.DLC) and (not base.isDemo or Config.download.demo) and (not self.isUpdate or Config.download.update) and ( base.key or Config.download.sansTitleKey or self.isUpdate or skipKeyCheck) and (len(Config.titleWhitelist) == 0 or self.id in Config.titleWhitelist) and self.id not in Config.titleBlacklist: if Config.shardIndex is not None and Config.shardCount is not None: if (int(self.id[0:13], 16) // 2) % Config.shardCount != Config.shardIndex: return False if Config.download.rankMin is not None: if base.rank is None or base.rank < Config.download.rankMin: return False if Config.download.rankMax is not None: if base.rank is None or base.rank > Config.download.rankMax: return False if Config.download.regions is not None and len(Config.download.regions) > 0: if not Config.download.hasRegion(base.regions): return False return True else: return False
def getTitleImage(request, response): if len(request.bits) < 3: return Server.Response404(request, response) id = request.bits[2] try: width = int(request.bits[3]) except BaseException: return Server.Response404(request, response) if width < 32 or width > 1024: return Server.Response404(request, response) if not Titles.contains(id): return Server.Response404(request, response) path = Titles.get(id).iconFile(width) or Titles.get(id).frontBoxArtFile( width) if not path: return Server.Response404(request, response) response.setMime(path) response.headers['Cache-Control'] = 'max-age=31536000' if os.path.isfile(path): with open(path, 'rb') as f: response.write(f.read()) return Server.Response500(request, response)
def getScreenshotImage(request, response): if len(request.bits) < 3: return Server.Response404(request, response) id = request.bits[2] try: i = int(request.bits[3]) except BaseException: return Server.Response404(request, response) if not Titles.contains(id): return Server.Response404(request, response) path = Titles.get(id).screenshotFile(i) if not path: return Server.Response404(request, response) response.setMime(path) response.headers['Cache-Control'] = 'max-age=31536000' if os.path.isfile(path): with open(path, 'rb') as f: response.write(f.read()) return Server.Response500(request, response)
def download(id): bits = id.split(',') version = None key = None if len(bits) == 1: id = bits[0].upper() elif len(bits) == 2: id = bits[0].upper() key = bits[1].strip() elif len(bits) == 3: id = bits[0].upper() key = bits[1].strip() version = bits[2].strip() else: Print.info('invalid args: ' + download) return False if key == '': key = None if version == '': version = None if len(id) != 16: raise IOError('Invalid title id format') if Titles.contains(id): title = Titles.get(id) cdn.downloadTitle(title.id.lower(), version, key or title.key) else: cdn.downloadTitle(id.lower(), version, key) return True
def scanLatestTitleUpdates(): nut.initTitles() nut.initFiles() for k, i in CDNSP.get_versionUpdates().items(): id = str(k).upper() version = str(i) if not Titles.contains(id): if len(id) != 16: Print.info('invalid title id: ' + id) continue continue t = Title() t.setId(id) Titles.set(id, t) Print.info('Found new title id: ' + str(id)) t = Titles.get(id) if str(t.version) != str(version): Print.info('new version detected for %s[%s] v%s' % (t.name or '', t.id or ('0' * 16), str(version))) t.setVersion(version, True) Titles.save()
def isUpdateAvailable(self): title = self.title() if self.titleId and str( title.version) is not None and str(self.version) < str( title.version) and str(title.version) != '0': return { 'id': title.id, 'baseId': title.baseId, 'currentVersion': str(self.version), 'newVersion': str(title.version) } if not title.isUpdate and not title.isDLC and Titles.contains( title.updateId): updateFile = self.getUpdateFile() if updateFile: return updateFile.isUpdateAvailable() updateTitle = Titles.get(title.updateId) if str(updateTitle.version) and str(updateTitle.version) != '0': return { 'id': updateTitle.id, 'baseId': title.baseId, 'currentVersion': None, 'newVersion': str(updateTitle.version) } return None
def get_name(titleId): titleId = titleId.upper() if Titles.contains(titleId): try: t = Titles.get(titleId) return (re.sub(r'[/\\:*?!"|???]+', "", unidecode.unidecode(t.name.strip())))[:70] except: pass return 'Unknown Title'
def sendTitleCard(channelId, titleId, nsp = None): if not Titles.contains(titleId): send(channelId, titleId) return title = Titles.get(titleId) titleBase = title.getBase() or title filename = None if nsp: filename = os.path.basename(nsp.fileName()) embed = discord.Embed(title=title.name or titleBase.name, description=title.intro, color=0x3498DB, url = "https://tinfoil.io/Title/" + titleId.upper(), timestamp = formatDate(title.releaseDate)) if filename: embed.add_field(name="File Name", value=filename, inline=False) else: embed.add_field(name="ID", value=titleId.upper(), inline=True) if title.isUpdate: embed.add_field(name="Type", value='Update', inline=True) elif title.isDLC: embed.add_field(name="Type", value='DLC', inline=True) else: embed.add_field(name="Type", value='Base', inline=True) if nsp: if nsp.getFileSize(): embed.add_field(name="Size", value=formatSize(nsp.getFileSize()), inline=True) else: if title.size: embed.add_field(name="Size", value=formatSize(title.size), inline=True) if title.releaseDate: embed.set_footer(text="Released") if title.iconUrl or titleBase.iconUrl: embed.set_thumbnail(url = title.iconUrl or titleBase.iconUrl) if nsp: ext = nsp.path.split('.')[-1].lower() #if nsp.version: # embed.add_field(name="Version", value=str(nsp.version), inline=True) if ext == 'nsz': try: embed.add_field(name="Compression", value='-' + str(100 - int(nsp.getCr())) + '%', inline=True) except: pass send(channelId, embed = embed)
def _ftpsync(url): if Config.reverse: q = queue.LifoQueue() else: q = queue.Queue() fileList = [] for f in Fs.driver.openDir(url).ls(): if f.isFile(): fileList.append(f.url) for path in fileList: try: #print('checking ' + path) nsp = Fs.Nsp() nsp.setPath(urllib.parse.unquote(path)) nsp.downloadPath = path if not nsp.titleId: continue if not Titles.contains(nsp.titleId) or ( not len(Titles.get(nsp.titleId).getFiles(path[-3:])) and Titles.get(nsp.titleId).isActive(skipKeyCheck=True)): if path[-3:] == 'nsx': if len(Titles.get(nsp.titleId).getFiles('nsp')) or len( Titles.get(nsp.titleId).getFiles('nsz')): continue q.put(nsp) except BaseException as e: Print.error(str(e)) #raise #TODO numThreads = Config.threads threads = [] s = Status.create(q.qsize(), 'Total File Pulls') if numThreads > 0: Print.info('creating pull threads, items: ' + str(q.qsize())) for i in range(numThreads): t = threading.Thread(target=pullWorker, args=[q, s]) t.daemon = True t.start() threads.append(t) for t in threads: t.join() else: pullWorker(q, s) s.close()
def unlock(self): #if not self.isOpen(): # self.open('r+b') if not Titles.contains(self.titleId): raise IOError('No title key found in database!') self.ticket().setTitleKeyBlock(int(Titles.get(self.titleId).key, 16)) Print.info('setting title key to ' + Titles.get(self.titleId).key) self.ticket().flush() self.close() self.hasValidTicket = True self.move()
def getFiles(request, response): r = {} for path, nsp in Nsps.files.items(): if Titles.contains(nsp.titleId): title = Titles.get(nsp.titleId) if title.baseId not in r: r[title.baseId] = {'base': [], 'dlc': [], 'update': []} if title.isDLC: r[title.baseId]['dlc'].append(nsp.dict()) elif title.isUpdate: r[title.baseId]['update'].append(nsp.dict()) else: r[title.baseId]['base'].append(nsp.dict()) response.write(json.dumps(r))
def download(id): bits = id.split(',') version = None key = None if len(bits) == 1: id = bits[0].upper() elif len(bits) == 2: id = bits[0].upper() key = bits[1].strip() elif len(bits) == 3: id = bits[0].upper() key = bits[1].strip() version = bits[2].strip() else: Print.info('invalid args: ' + download) return False if key == '': key = None if version == '': version = None if len(id) != 16: raise IOError('Invalid title id format') if Titles.contains(id): title = Titles.get(id) if version == None: version = title.lastestVersion() if version == None: if not title.key: Titles.erase(id) return False CDNSP.download_game(title.id.lower(), version or title.lastestVersion(), key or title.key, True, '', True) else: CDNSP.download_game(id.lower(), version or CDNSP.get_version(id.lower()), key, True, '', True) return True
def scanLatestTitleUpdates(): global versionHistory initTitles() initFiles() now = datetime.datetime.now() today = now.strftime("%Y-%m-%d") try: with open('titledb/versions.json', 'r') as f: for titleId, vers in json.loads(f.read()).items(): for ver, date in vers.items(): setVersionHistory(titleId, ver, date) except BaseException: pass if not hasCdn: return for k, i in cdn.hacVersionList().items(): id = str(k).upper() version = str(i) if not Titles.contains(id): if len(id) != 16: Print.info('invalid title id: ' + id) continue t = Titles.get(id) if t.isUpdate: setVersionHistory(Title.getBaseId(id), version, today) else: setVersionHistory(id, version, today) if str(t.version) != str(version): Print.info('new version detected for %s[%s] v%s' % (t.name or '', t.id or ('0' * 16), str(version))) t.setVersion(version, True) Titles.save() try: with open('titledb/versions.json', 'w') as outfile: json.dump(versionHistory, outfile, indent=4, sort_keys=True) except BaseException as e: Print.info(str(e))
def scanDLC(id, showErr=True, dlcStatus=None): id = id.upper() title = Titles.get(id) baseDlc = Title.baseDlcId(id) for i in range(0x1FF): scanId = format(baseDlc + i, 'X').zfill(16) if Titles.contains(scanId): continue ver = CDNSP.get_version(scanId.lower()) if ver != None: t = Title() t.setId(scanId) Titles.set(scanId, t) Titles.save() Print.info('Found new DLC ' + str(title.name) + ' : ' + scanId) elif showErr: Print.info('nothing found at ' + scanId + ', ' + str(ver)) if dlcStatus: dlcStatus.add()
def getFiles(request, response): global last_scan_datetime if datetime.now() > last_scan_datetime + timedelta( seconds=Config.scanDebounceSeconds): nut.scan() last_scan_datetime = datetime.now() r = {} for path, nsp in Nsps.files.items(): if Titles.contains(nsp.titleId): title = Titles.get(nsp.titleId) if not title.baseId in r: r[title.baseId] = {'base': [], 'dlc': [], 'update': []} if title.isDLC: r[title.baseId]['dlc'].append(nsp.dict()) elif title.isUpdate: r[title.baseId]['update'].append(nsp.dict()) else: r[title.baseId]['base'].append(nsp.dict()) response.write(json.dumps(r))
def scanBaseThread(baseStatus): while Config.isRunning: try: id = getRandomTitleId() if Titles.contains(id): continue ver = CDNSP.get_version(id.lower()) if ver != None: Print.info('Found new base ' + id) t = Title() t.setId(id) Titles.set(id, t) Titles.save() baseStatus.add() except BaseException as e: print('exception: ' + str(e))
def getAddOns(titleId, shop_id=3): url = 'https://superfly.hac.%s.d4c.nintendo.net/v1/a/%s/dv' % (Config.cdn.environment, titleId) j = makeJsonRequest('GET', url, {}, '%d/a/%s/dv.json' % (shop_id, titleId), force = False) lst = [] if not j: return lst for i in j: id = i['title_id'].upper() if not Titles.contains(id): Print.info('New DLC found: ' + id) title = Titles.get(id, None, None) title.setVersion(int(i['version'])) lst.append(id) return lst
def removeTitleRights(self): if not Titles.contains(self.titleId): raise IOError('No title key found in database! ' + self.titleId) ticket = self.ticket() masterKeyRev = ticket.getMasterKeyRevision() titleKeyDec = Keys.decryptTitleKey( ticket.getTitleKeyBlock().to_bytes(16, byteorder='big'), Keys.getMasterKeyIndex(masterKeyRev)) rightsId = ticket.getRightsId() Print.info('rightsId =\t' + hex(rightsId)) Print.info('titleKeyDec =\t' + str(hx(titleKeyDec))) Print.info('masterKeyRev =\t' + hex(masterKeyRev)) for nca in self: if type(nca) == Nca: if nca.header.getCryptoType2() != masterKeyRev: pass raise IOError('Mismatched masterKeyRevs!') ticket.setRightsId(0) for nca in self: if type(nca) == Nca: if nca.header.getRightsId() == 0: continue kek = Keys.keyAreaKey(Keys.getMasterKeyIndex(masterKeyRev), nca.header.keyIndex) Print.info('writing masterKeyRev for %s, %d' % (str(nca._path), masterKeyRev)) Print.info('kek =\t' + hx(kek).decode()) crypto = aes128.AESECB(kek) encKeyBlock = crypto.encrypt(titleKeyDec * 4) nca.header.setRightsId(0) nca.header.setKeyBlock(encKeyBlock) Hex.dump(encKeyBlock)
def getBannerImage(request, response): if len(request.bits) < 3: return Server.Response404(request, response) id = request.bits[2] if not Titles.contains(id): return Server.Response404(request, response) path = Titles.get(id).bannerFile() if not path: return Server.Response404(request, response) response.setMime(path) response.headers['Cache-Control'] = 'max-age=31536000' if os.path.isfile(path): with open(path, 'rb') as f: response.write(f.read()) return Server.Response500(request, response)
def updateVersions(force=True): initTitles() initFiles() i = 0 for k, t in tqdm(Titles.items()): if force or t.version is None: if t.isActive(): v = t.lastestVersion(True) Print.info("%s[%s] v = %s" % (str(t.name), str(t.id), str(v))) for t in list(Titles.data().values()): if not t.isUpdate and not t.isDLC and t.updateId and t.updateId and not Titles.contains( t.updateId): u = Title.Title() u.setId(t.updateId) if u.lastestVersion(): Titles.set(t.updateId, u) Print.info("%s[%s] FOUND" % (str(t.name), str(u.id))) Titles.save()
def getBaseName(self): baseId = getBaseId(self.id) if Titles.contains(baseId): return (Titles.get(baseId).name or '').replace('\n', ' ') return ''
def getName(self): baseId = getBaseId(self.id) if hasattr(self, 'isUpdate') and self.isUpdate and Titles.contains(baseId): return (Titles.get(baseId).name or '').replace('\n', ' ') return (self.name or '').replace('\n', ' ')
def isUnlockable(self, reunlock=False): return (not self.hasValidTicket or reunlock) and self.titleId and Titles.contains( self.titleId) and Titles.get(self.titleId).key
if args.Z: nut.updateVersions(True) if args.z: nut.updateVersions(False) if args.V: nut.scanLatestTitleUpdates() nut.export('titledb/versions.txt', ['id', 'rightsId', 'version']) if args.scrape_title: nut.initTitles() nut.initFiles() if not Titles.contains(args.scrape_title): Print.error('Could not find title ' + args.scrape_title) else: Titles.get(args.scrape_title).scrape(False) Titles.save() # Print.info(repr(Titles.get(args.scrape_title).__dict__)) pprint.pprint(Titles.get(args.scrape_title).__dict__) if args.download_all: nut.downloadAll() Titles.save() if args.gen_tinfoil_titles: genTinfoilTitles()
def setMasterKeyRev(self, newMasterKeyRev): if not Titles.contains(self.titleId): raise IOError('No title key found in database! ' + self.titleId) ticket = self.ticket() masterKeyRev = ticket.getMasterKeyRevision() titleKey = ticket.getTitleKeyBlock() newTitleKey = Keys.changeTitleKeyMasterKey( titleKey.to_bytes(16, byteorder='big'), Keys.getMasterKeyIndex(masterKeyRev), Keys.getMasterKeyIndex(newMasterKeyRev)) rightsId = ticket.getRightsId() if rightsId != 0: raise IOError('please remove titlerights first') if (newMasterKeyRev == None and rightsId == 0) or masterKeyRev == newMasterKeyRev: Print.info('Nothing to do') return Print.info('rightsId =\t' + hex(rightsId)) Print.info('titleKey =\t' + str(hx(titleKey.to_bytes(16, byteorder='big')))) Print.info('newTitleKey =\t' + str(hx(newTitleKey))) Print.info('masterKeyRev =\t' + hex(masterKeyRev)) for nca in self: if type(nca) == Nca: if nca.header.getCryptoType2() != masterKeyRev: pass raise IOError('Mismatched masterKeyRevs!') ticket.setMasterKeyRevision(newMasterKeyRev) ticket.setRightsId((ticket.getRightsId() & 0xFFFFFFFFFFFFFFFF0000000000000000) + newMasterKeyRev) ticket.setTitleKeyBlock(int.from_bytes(newTitleKey, 'big')) for nca in self: if type(nca) == Nca: if nca.header.getCryptoType2() != newMasterKeyRev: Print.info('writing masterKeyRev for %s, %d -> %s' % (str(nca._path), nca.header.getCryptoType2(), str(newMasterKeyRev))) encKeyBlock = nca.header.getKeyBlock() if sum(encKeyBlock) != 0: key = Keys.keyAreaKey( Keys.getMasterKeyIndex(masterKeyRev), nca.header.keyIndex) Print.info('decrypting with %s (%d, %d)' % (str(hx(key)), Keys.getMasterKeyIndex(masterKeyRev), nca.header.keyIndex)) crypto = aes128.AESECB(key) decKeyBlock = crypto.decrypt(encKeyBlock) key = Keys.keyAreaKey( Keys.getMasterKeyIndex(newMasterKeyRev), nca.header.keyIndex) Print.info('encrypting with %s (%d, %d)' % (str(hx(key)), Keys.getMasterKeyIndex(newMasterKeyRev), nca.header.keyIndex)) crypto = aes128.AESECB(key) reEncKeyBlock = crypto.encrypt(decKeyBlock) nca.header.setKeyBlock(reEncKeyBlock) if newMasterKeyRev >= 3: nca.header.setCryptoType(2) nca.header.setCryptoType2(newMasterKeyRev) else: nca.header.setCryptoType(newMasterKeyRev) nca.header.setCryptoType2(0)
def getBase(self): baseId = getBaseId(self.id) if Titles.contains(baseId): return Titles.get(baseId) return None