def _ftpsync(url):
if Config.reverse:
q = queue.LifoQueue()
else:
q = queue.Queue()
fileList = []
for f in Fs.driver.openDir(url).ls():
if f.isFile():
fileList.append(f.url)
for path in fileList:
try:
if path.split('.')[-1].lower() not in ('nsx', 'nsz', 'nsp', 'xci'):
continue
unq = urllib.parse.unquote(path)
nsp = Fs.factory(unq, unq, None)
nsp.downloadPath = path
if not nsp.titleId:
continue
title = Titles.get(nsp.titleId)
if not title.isActive(skipKeyCheck=True):
continue
files = title.getFiles(path[-3:])
files = [x for x in files if int(x.version) >= int(nsp.version)]
if not len(files):
if path[-3:] == 'nsx':
if len(Titles.get(nsp.titleId).getFiles('nsp')) or len(
Titles.get(nsp.titleId).getFiles('nsz')):
continue
q.put(nsp)
except BaseException as e:
Print.error(str(e))
# raise #TODO
numThreads = Config.threads
threads = []
s = Status.create(q.qsize(), 'Total File Pulls')
if numThreads > 0:
Print.info('creating pull threads, items: ' + str(q.qsize()))
for i in range(numThreads):
t = threading.Thread(target=pullWorker, args=[q, s])
t.daemon = True
t.start()
threads.append(t)
for t in threads:
t.join()
else:
pullWorker(q, s)
s.close()
def importRegion(region='US', language='en'):
if not region in Config.regionLanguages(
) or language not in Config.regionLanguages()[region]:
Print.error('Could not locate %s/%s !' % (region, language))
return False
for region2 in Config.regionLanguages():
for language2 in Config.regionLanguages()[region2]:
for nsuId, regionTitle in Titles.data(region2, language2).items():
if not regionTitle.id:
continue
title = Titles.get(regionTitle.id, None, None)
title.importFrom(regionTitle, region2, language2)
for region2 in Config.regionLanguages():
for language2 in Config.regionLanguages()[region2]:
if language2 != language:
continue
for nsuId, regionTitle in Titles.data(region2, language2).items():
if not regionTitle.id:
continue
title = Titles.get(regionTitle.id, None, None)
title.importFrom(regionTitle, region2, language2)
for nsuId, regionTitle in Titles.data(region, language).items():
if not regionTitle.id:
continue
title = Titles.get(regionTitle.id, None, None)
title.importFrom(regionTitle, region, language)
Titles.loadTxtDatabases()
Titles.save()
def getTitleImage(request, response):
if len(request.bits) < 3:
return Server.Response404(request, response)
id = request.bits[2]
try:
width = int(request.bits[3])
except BaseException:
return Server.Response404(request, response)
if width < 32 or width > 1024:
return Server.Response404(request, response)
if not Titles.contains(id):
return Server.Response404(request, response)
path = Titles.get(id).iconFile(width) or Titles.get(id).frontBoxArtFile(
width)
if not path:
return Server.Response404(request, response)
response.setMime(path)
response.headers['Cache-Control'] = 'max-age=31536000'
if os.path.isfile(path):
with open(path, 'rb') as f:
response.write(f.read())
return Server.Response500(request, response)
def importRegion(region='US', language='en', save=True):
if region not in Config.regionLanguages() or language not in Config.regionLanguages()[region]:
Print.info('Could not locate %s/%s !' % (region, language))
return False
Hook.call("import.pre", region, language)
regionLanguages = []
for region2 in Config.regionLanguages():
for language2 in Config.regionLanguages()[region2]:
regionLanguages.append(RegionLanguage(region2, language2, region, language))
for rl in sorted(regionLanguages):
data = Titles.data(rl.region, rl.language)
for nsuId in sorted(data.keys(), reverse=True):
regionTitle = data[nsuId]
if not regionTitle.id:
continue
try:
for tid in regionTitle.ids:
title = Titles.get(tid, None, None)
title.importFrom(regionTitle, rl.region, rl.language, preferredRegion=region, preferredLanguage=language)
except:
title = Titles.get(regionTitle.id, None, None)
title.importFrom(regionTitle, rl.region, rl.language, preferredRegion=region, preferredLanguage=language)
Titles.loadTxtDatabases()
Hook.call("import.post", region, language)
if save:
Titles.save()
def fileName(self):
bt = None
if not self.titleId in Titles.keys():
if not Title.getBaseId(self.titleId) in Titles.keys():
Print.info('could not find title key for ' +
str(self.titleId) + ' or ' +
str(Title.getBaseId(self.titleId)))
return None
bt = Titles.get(Title.getBaseId(self.titleId))
t = Title()
t.loadCsv(self.titleId + '0000000000000000|0000000000000000|' +
bt.name)
else:
t = Titles.get(self.titleId)
if not t:
Print.error('could not find title id ' + str(self.titleId))
return None
try:
if not t.baseId in Titles.keys():
Print.info('could not find baseId for ' + self.path)
return None
except BaseException as e:
print('exception: could not find title id ' +
str(self.titleId) + ' ' + str(e))
return None
bt = Titles.get(t.baseId)
if t.isDLC:
format = Config.paths.getTitleDLC(not self.hasValidTicket)
elif t.isDemo:
if t.idExt != 0:
format = Config.paths.getTitleDemoUpdate(
not self.hasValidTicket)
else:
format = Config.paths.getTitleDemo(not self.hasValidTicket)
elif t.idExt != 0:
format = Config.paths.getTitleUpdate(not self.hasValidTicket)
else:
format = Config.paths.getTitleBase(not self.hasValidTicket)
format = format.replace('{id}', self.cleanFilename(t.id))
format = format.replace('{region}',
self.cleanFilename(t.getRegion() or ''))
format = format.replace('{name}', self.cleanFilename(t.getName()
or ''))
format = format.replace('{version}', str(self.getVersion() or 0))
format = format.replace('{baseId}', self.cleanFilename(bt.id))
baseName = self.cleanFilename(bt.getName() or '')
result = format.replace('{baseName}', baseName)
while (len(os.path.basename(result).encode('utf-8')) > 240
and len(baseName) > 3):
baseName = baseName[:-1]
result = format.replace('{baseName}', baseName)
return result
def scrapeThread(st, delta=True):
while scrapeQueue.qsize() > 0:
titleId = scrapeQueue.get()
try:
Titles.get(titleId).scrape(delta)
st.add()
except BaseException as e:
Print.error(str(e))
Print.info('thread exit')
def scrapeThread(id, delta=True):
size = len(Titles.titles) // scrapeThreads
st = Status.create(size, 'Thread ' + str(id))
for i, titleId in enumerate(Titles.titles.keys()):
try:
if (i - id) % scrapeThreads == 0:
Titles.get(titleId).scrape(delta)
st.add()
except BaseException as e:
Print.error(str(e))
st.close()
def _ftpsync(url):
if Config.reverse:
q = queue.LifoQueue()
else:
q = queue.Queue()
fileList = []
for f in Fs.driver.openDir(url).ls():
if f.isFile():
fileList.append(f.url)
for path in fileList:
try:
#print('checking ' + path)
nsp = Fs.Nsp()
nsp.setPath(urllib.parse.unquote(path))
nsp.downloadPath = path
if not nsp.titleId:
continue
if not Titles.contains(nsp.titleId) or (
not len(Titles.get(nsp.titleId).getFiles(path[-3:]))
and Titles.get(nsp.titleId).isActive(skipKeyCheck=True)):
if path[-3:] == 'nsx':
if len(Titles.get(nsp.titleId).getFiles('nsp')) or len(
Titles.get(nsp.titleId).getFiles('nsz')):
continue
q.put(nsp)
except BaseException as e:
Print.error(str(e))
#raise #TODO
numThreads = Config.threads
threads = []
s = Status.create(q.qsize(), 'Total File Pulls')
if numThreads > 0:
Print.info('creating pull threads, items: ' + str(q.qsize()))
for i in range(numThreads):
t = threading.Thread(target=pullWorker, args=[q, s])
t.daemon = True
t.start()
threads.append(t)
for t in threads:
t.join()
else:
pullWorker(q, s)
s.close()
def unlock(self):
#if not self.isOpen():
# self.open('r+b')
if not Titles.contains(self.titleId):
raise IOError('No title key found in database!')
self.ticket().setTitleKeyBlock(int(Titles.get(self.titleId).key, 16))
Print.info('setting title key to ' + Titles.get(self.titleId).key)
self.ticket().flush()
self.close()
self.hasValidTicket = True
self.move()
def post(args):
if not args.import_title_keys:
return
nut.initTitles()
nut.initFiles()
with open(args.import_title_keys, 'r') as f:
for line in f.read().split('\n'):
if '=' not in line:
continue
try:
rightsId, key = line.split('=')
rightsId = rightsId.strip()
titleId = rightsId[0:16]
key = key.strip()
title = Titles.get(titleId)
nsp = title.getLatestNsp()
nsz = title.getLatestNsz()
print(nsp)
if not nsp and not nsz:
Print.info('title import: new title detected: %s - %s' %
(title.id, title.name))
elif not title.key:
Print.info(
'title import: new title key detected: %s - %s' %
(title.id, title.name))
title.rightsId = rightsId
title.key = key
except:
raise
Titles.save()
def matchDemos():
nut.initTitles()
nut.initFiles()
orphans = {}
Titles.loadTxtDatabases()
for nsuId, titleId in Titles.nsuIdMap.items():
for region, languages in Config.regionLanguages().items():
for language in languages:
if nsuId:
title = Titles.get(str(nsuId), region, language)
title.id = titleId
for region, languages in Config.regionLanguages().items():
for language in languages:
for nsuId, rt in Titles.data(region, language).items():
if rt.id:
continue
orphans[nsuId] = rt.name
Titles.saveRegion(region, language)
for nsuId, name in orphans.items():
print(str(nsuId) + '|' + str(name))
def download(id):
bits = id.split(',')
version = None
key = None
if len(bits) == 1:
id = bits[0].upper()
elif len(bits) == 2:
id = bits[0].upper()
key = bits[1].strip()
elif len(bits) == 3:
id = bits[0].upper()
key = bits[1].strip()
version = bits[2].strip()
else:
Print.info('invalid args: ' + download)
return False
if key == '':
key = None
if version == '':
version = None
if len(id) != 16:
raise IOError('Invalid title id format')
if Titles.contains(id):
title = Titles.get(id)
cdn.downloadTitle(title.id.lower(), version, key or title.key)
else:
cdn.downloadTitle(id.lower(), version, key)
return True
def isUpdateAvailable(self):
title = self.title()
if self.titleId and str(
title.version) is not None and str(self.version) < str(
title.version) and str(title.version) != '0':
return {
'id': title.id,
'baseId': title.baseId,
'currentVersion': str(self.version),
'newVersion': str(title.version)
}
if not title.isUpdate and not title.isDLC and Titles.contains(
title.updateId):
updateFile = self.getUpdateFile()
if updateFile:
return updateFile.isUpdateAvailable()
updateTitle = Titles.get(title.updateId)
if str(updateTitle.version) and str(updateTitle.version) != '0':
return {
'id': updateTitle.id,
'baseId': title.baseId,
'currentVersion': None,
'newVersion': str(updateTitle.version)
}
return None
def scanLatestTitleUpdates():
nut.initTitles()
nut.initFiles()
for k, i in CDNSP.get_versionUpdates().items():
id = str(k).upper()
version = str(i)
if not Titles.contains(id):
if len(id) != 16:
Print.info('invalid title id: ' + id)
continue
continue
t = Title()
t.setId(id)
Titles.set(id, t)
Print.info('Found new title id: ' + str(id))
t = Titles.get(id)
if str(t.version) != str(version):
Print.info('new version detected for %s[%s] v%s' %
(t.name or '', t.id or ('0' * 16), str(version)))
t.setVersion(version, True)
Titles.save()
def downloadThread(i):
Print.info('starting thread ' + str(i))
global status
while Config.isRunning:
try:
id = Titles.queue.shift()
if id and Titles.contains(id):
activeDownloads[i] = 1
t = Titles.get(id)
path = CDNSP.download_game(t.id.lower(), t.lastestVersion(),
t.key, True, '', True)
if os.path.isfile(path):
nsp = Fs.Nsp(path, None)
nsp.move()
Nsps.files[nsp.path] = nsp
Nsps.save()
status.add()
activeDownloads[i] = 0
else:
time.sleep(1)
except KeyboardInterrupt:
pass
except BaseException as e:
Print.error(str(e))
activeDownloads[i] = 0
Print.info('ending thread ' + str(i))
def downloadThread(i):
if not hasCdn:
return
Print.info('starting thread ' + str(i))
global status
while Config.isRunning and not Titles.queue.empty():
try:
id = Titles.queue.shift()
if id and Titles.contains(id):
activeDownloads[i] = 1
t = Titles.get(id)
path = cdn.downloadTitle(t.id.lower(), None, t.key)
if path and os.path.isfile(path):
nsp = Fs.Nsp(path, None)
nsp.move()
Nsps.save()
if status is not None:
status.add()
activeDownloads[i] = 0
else:
time.sleep(1)
except KeyboardInterrupt:
pass
except BaseException as e:
Print.error('downloadThread exception: ' + str(e))
traceback.print_exc(file=sys.stdout)
activeDownloads[i] = 0
Print.info('ending thread ' + str(i))
def getFrontArtBoxImage(request, response):
return getTitleImage(request, response)
if len(request.bits) < 3:
return Server.Response404(request, response)
id = request.bits[2]
#width = int(request.bits[3])
#if width < 32 or width > 512:
# return Server.Response404(request, response)
if not Titles.contains(id):
return Server.Response404(request, response)
path = Titles.get(id).frontBoxArtFile()
if not path:
return Server.Response404(request, response)
response.setMime(path)
response.headers['Cache-Control'] = 'max-age=31536000'
if os.path.isfile(path):
with open(path, 'rb') as f:
response.write(f.read())
return Server.Response500(request, response)
def getScreenshotImage(request, response):
if len(request.bits) < 3:
return Server.Response404(request, response)
id = request.bits[2]
try:
i = int(request.bits[3])
except BaseException:
return Server.Response404(request, response)
if not Titles.contains(id):
return Server.Response404(request, response)
path = Titles.get(id).screenshotFile(i)
if not path:
return Server.Response404(request, response)
response.setMime(path)
response.headers['Cache-Control'] = 'max-age=31536000'
if os.path.isfile(path):
with open(path, 'rb') as f:
response.write(f.read())
return Server.Response500(request, response)
def isActive(self, skipKeyCheck=False): if self.id[0:13] == '0100000000000': return False base = self if self.isDLC or self.isUpdate: baseId = getBaseId(self.id) if Titles.contains(baseId): base = Titles.get(baseId) if (self.isDLC or self.isUpdate or Config.download.base) and (not self.isDLC or Config.download.DLC) and (not base.isDemo or Config.download.demo) and (not self.isUpdate or Config.download.update) and ( base.key or Config.download.sansTitleKey or self.isUpdate or skipKeyCheck) and (len(Config.titleWhitelist) == 0 or self.id in Config.titleWhitelist) and self.id not in Config.titleBlacklist: if Config.shardIndex is not None and Config.shardCount is not None: if (int(self.id[0:13], 16) // 2) % Config.shardCount != Config.shardIndex: return False if Config.download.rankMin is not None: if base.rank is None or base.rank < Config.download.rankMin: return False if Config.download.rankMax is not None: if base.rank is None or base.rank > Config.download.rankMax: return False if Config.download.regions is not None and len(Config.download.regions) > 0: if not Config.download.hasRegion(base.regions): return False return True else: return False
def restore(self):
header = self.getVerifiedHeader()
if not header:
raise IOError('could not restore nca header for %s - %s' %
(self.titleId, str(self.f._path)))
if not self.hasTitleRights():
if header[0x30:0x40] != b'\x00' * 0x10:
self.rightsId = hx(header[0x30:0x40])
key = hx(
Keys.encryptTitleKey(
self.key(), max(max(header[0x6], header[0x20]) - 1,
0))).decode().upper()
#key = hx(self.key()).decode().upper()
title = Titles.get(self.titleId)
if title.key and title.key != key:
raise IOError(
'nca title key does not match database: %s vs %s' %
(title.key, key))
elif not title.key:
title.key = key
else:
self.rightsId = b'0' * 32
self.seek(0x200)
self.write(header)
return True
def exportVerifiedKeys(fileName):
nut.initTitles()
with open(fileName, 'w') as f:
f.write('id|key|version\n')
for tid, key in blockchain.blockchain.export().items():
title = Titles.get(tid)
if title and title.rightsId:
f.write(str(title.rightsId) + '|' + str(key) + '|' + str(title.version) + '\n')
def getInfo(request, response):
try:
nsp = Nsps.getByTitleId(request.bits[2])
t = Titles.get(request.bits[2]).__dict__
t['size'] = nsp.getFileSize()
t['mtime'] = nsp.getFileModified()
response.write(json.dumps(t))
except BaseException as e:
response.write(json.dumps({'success': False, 'message': str(e)}))
def get_name(titleId): titleId = titleId.upper() if Titles.contains(titleId): try: t = Titles.get(titleId) return (re.sub(r'[/\\:*?!"|???]+', "", unidecode.unidecode(t.name.strip())))[:70] except: pass return 'Unknown Title'
def title(self):
if not self.titleId:
raise IOError('NSP no titleId set')
if self.titleId in Titles.keys():
return Titles.get(self.titleId)
t = Title.Title()
t.setId(self.titleId)
Titles.data()[self.titleId] = t
return t
def sendTitleCard(channelId, titleId, nsp = None):
if not Titles.contains(titleId):
send(channelId, titleId)
return
title = Titles.get(titleId)
titleBase = title.getBase() or title
filename = None
if nsp:
filename = os.path.basename(nsp.fileName())
embed = discord.Embed(title=title.name or titleBase.name, description=title.intro, color=0x3498DB, url = "https://tinfoil.io/Title/" + titleId.upper(), timestamp = formatDate(title.releaseDate))
if filename:
embed.add_field(name="File Name", value=filename, inline=False)
else:
embed.add_field(name="ID", value=titleId.upper(), inline=True)
if title.isUpdate:
embed.add_field(name="Type", value='Update', inline=True)
elif title.isDLC:
embed.add_field(name="Type", value='DLC', inline=True)
else:
embed.add_field(name="Type", value='Base', inline=True)
if nsp:
if nsp.getFileSize():
embed.add_field(name="Size", value=formatSize(nsp.getFileSize()), inline=True)
else:
if title.size:
embed.add_field(name="Size", value=formatSize(title.size), inline=True)
if title.releaseDate:
embed.set_footer(text="Released")
if title.iconUrl or titleBase.iconUrl:
embed.set_thumbnail(url = title.iconUrl or titleBase.iconUrl)
if nsp:
ext = nsp.path.split('.')[-1].lower()
#if nsp.version:
# embed.add_field(name="Version", value=str(nsp.version), inline=True)
if ext == 'nsz':
try:
embed.add_field(name="Compression", value='-' + str(100 - int(nsp.getCr())) + '%', inline=True)
except:
pass
send(channelId, embed = embed)
def getFiles(request, response):
r = {}
for path, nsp in Nsps.files.items():
if Titles.contains(nsp.titleId):
title = Titles.get(nsp.titleId)
if title.baseId not in r:
r[title.baseId] = {'base': [], 'dlc': [], 'update': []}
if title.isDLC:
r[title.baseId]['dlc'].append(nsp.dict())
elif title.isUpdate:
r[title.baseId]['update'].append(nsp.dict())
else:
r[title.baseId]['base'].append(nsp.dict())
response.write(json.dumps(r))
def refreshTable(self):
self.tableWidget.setRowCount(len(Nsps.files))
i = 0
for k, f in Nsps.files.items():
title = f.title()
if f.path.endswith('.nsx'):
continue
self.tableWidget.setItem(i,0, QTableWidgetItem(Titles.get(f.titleId).getName()))
self.tableWidget.setItem(i,1, QTableWidgetItem(str(f.titleId)))
self.tableWidget.setItem(i,2, QTableWidgetItem("UPD" if title.isUpdate else ("DLC" if title.isDLC else "BASE")))
self.tableWidget.setItem(i,3, QTableWidgetItem(approximate_size(f.fileSize or title.size)))
i = i + 1
self.tableWidget.setRowCount(i)
def download(id):
bits = id.split(',')
version = None
key = None
if len(bits) == 1:
id = bits[0].upper()
elif len(bits) == 2:
id = bits[0].upper()
key = bits[1].strip()
elif len(bits) == 3:
id = bits[0].upper()
key = bits[1].strip()
version = bits[2].strip()
else:
Print.info('invalid args: ' + download)
return False
if key == '':
key = None
if version == '':
version = None
if len(id) != 16:
raise IOError('Invalid title id format')
if Titles.contains(id):
title = Titles.get(id)
if version == None:
version = title.lastestVersion()
if version == None:
if not title.key:
Titles.erase(id)
return False
CDNSP.download_game(title.id.lower(), version
or title.lastestVersion(), key or title.key, True,
'', True)
else:
CDNSP.download_game(id.lower(), version
or CDNSP.get_version(id.lower()), key, True, '',
True)
return True
def scanLatestTitleUpdates():
global versionHistory
initTitles()
initFiles()
now = datetime.datetime.now()
today = now.strftime("%Y-%m-%d")
try:
with open('titledb/versions.json', 'r') as f:
for titleId, vers in json.loads(f.read()).items():
for ver, date in vers.items():
setVersionHistory(titleId, ver, date)
except BaseException:
pass
if not hasCdn:
return
for k, i in cdn.hacVersionList().items():
id = str(k).upper()
version = str(i)
if not Titles.contains(id):
if len(id) != 16:
Print.info('invalid title id: ' + id)
continue
t = Titles.get(id)
if t.isUpdate:
setVersionHistory(Title.getBaseId(id), version, today)
else:
setVersionHistory(id, version, today)
if str(t.version) != str(version):
Print.info('new version detected for %s[%s] v%s' %
(t.name or '', t.id or ('0' * 16), str(version)))
t.setVersion(version, True)
Titles.save()
try:
with open('titledb/versions.json', 'w') as outfile:
json.dump(versionHistory, outfile, indent=4, sort_keys=True)
except BaseException as e:
Print.info(str(e))
def getVerifiedHeader(self):
self.seek(0x200)
buffer = bytearray(self.read(0x200))
if self.verifyBuffer(buffer):
return buffer
for gameCardValue in [0, 1]:
buffer[0x04] = gameCardValue
if self.verifyBuffer(buffer):
Print.info('isGameCard = %d' % gameCardValue)
return buffer
if self.hasTitleRights():
return None
title = Titles.get(self.titleId)
'''
if title.rightsId:
for gameCardValue in [0, 1]:
buffer[0x04] = gameCardValue
#return False
'''
for gameCardValue in [0, 1]:
buffer[0x04] = gameCardValue
for keyGen in Keys.getKeyGens():
buffer = self.setRightsIdBuffer(buffer, keyGen)
if self.verifyBuffer(buffer):
Print.info('Title Rights: isGameCard = %d, keyGen = %d' %
(gameCardValue, keyGen))
return buffer
for gameCardValue in [0, 1]:
buffer[0x04] = gameCardValue
for keyGen in Keys.getKeyGens():
buffer = self.setStandardCryptoBuffer(buffer, keyGen)
if self.verifyBuffer(buffer):
Print.info(
'Standard Crypto: isGameCard = %d, keyGen = %d' %
(gameCardValue, keyGen))
return buffer
return None