def downloadAll(wait = True):
global activeDownloads
global status
try:
for k,t in Titles.items():
if t.isUpdateAvailable() and not t.retailOnly and (t.isDLC or t.isUpdate or Config.download.base) and (not t.isDLC or Config.download.DLC) and (not t.isDemo or Config.download.demo) and (not t.isUpdate or Config.download.update) and (t.key or Config.download.sansTitleKey) and (len(titleWhitelist) == 0 or t.id in titleWhitelist) and t.id not in titleBlacklist:
if not t.id or t.id == '0' * 16 or (t.isUpdate and t.lastestVersion() in [None, '0']):
#Print.warning('no valid id? ' + str(t.path))
continue
if not t.lastestVersion():
Print.info('Could not get version for ' + str(t.name))
continue
Titles.queue.add(t.id)
Titles.save()
status = Status.create(Titles.queue.size(), 'Total Download')
startDownloadThreads()
while wait and (not Titles.queue.empty() or sum(activeDownloads) > 0):
time.sleep(1)
except KeyboardInterrupt:
pass
except BaseException as e:
Print.error(str(e))
if status:
status.close()
def scanLatestTitleUpdates():
initTitles()
initFiles()
for k, i in CDNSP.get_versionUpdates().items():
id = str(k).upper()
version = str(i)
if not Titles.contains(id):
if len(id) != 16:
Print.info('invalid title id: ' + id)
continue
continue
t = Title()
t.setId(id)
Titles.set(id, t)
Print.info('Found new title id: ' + str(id))
t = Titles.get(id)
if str(t.version) != str(version):
Print.info('new version detected for %s[%s] v%s' %
(t.name or '', t.id or ('0' * 16), str(version)))
t.setVersion(version, True)
Titles.save()
def importRegion(region='US', language='en'):
if not region in Config.regionLanguages(
) or language not in Config.regionLanguages()[region]:
Print.error('Could not locate %s/%s !' % (region, language))
return False
for region2 in Config.regionLanguages():
for language2 in Config.regionLanguages()[region2]:
for nsuId, regionTitle in Titles.data(region2, language2).items():
if not regionTitle.id:
continue
title = Titles.get(regionTitle.id, None, None)
title.importFrom(regionTitle, region2, language2)
for region2 in Config.regionLanguages():
for language2 in Config.regionLanguages()[region2]:
if language2 != language:
continue
for nsuId, regionTitle in Titles.data(region2, language2).items():
if not regionTitle.id:
continue
title = Titles.get(regionTitle.id, None, None)
title.importFrom(regionTitle, region2, language2)
for nsuId, regionTitle in Titles.data(region, language).items():
if not regionTitle.id:
continue
title = Titles.get(regionTitle.id, None, None)
title.importFrom(regionTitle, region, language)
Titles.loadTxtDatabases()
Titles.save()
def genTinfoilTitles():
initTitles()
initFiles()
for region, languages in Config.regionLanguages().items():
for language in languages:
importRegion(region, language)
Titles.save('titledb/titles.%s.%s.json' % (region, language))
def scan(): global hasScanned if hasScanned: return hasScanned = True initTitles() initFiles() Nsps.scan(Config.paths.scan) Titles.save()
def genTinfoilTitles():
initTitles()
initFiles()
for region, languages in Config.regionLanguages().items():
for language in languages:
importRegion(region, language)
Titles.save('titledb/titles.%s.%s.json' % (region, language))
#Print.info('%s - %s' % (region, language))
scanLatestTitleUpdates()
export('titledb/versions.txt', ['id', 'version'])
def refresh():
initTitles()
initFiles()
for k, f in Nsps.files.items():
try:
f.open()
f.readMeta()
f.close()
except:
raise
pass
Titles.save()
def scan():
global hasScanned
#if hasScanned:
# return
hasScanned = True
initTitles()
initFiles()
refreshRegions()
importRegion(Config.region, Config.language)
r = Nsps.scan(Config.paths.scan)
Titles.save()
return r
def refresh(titleRightsOnly = False):
initTitles()
initFiles()
for k, f in Nsps.files.items():
try:
if titleRightsOnly:
title = Titles.get(f.titleId)
if title and title.rightsId:
continue
print(f.path)
f.open()
f.readMeta()
f.close()
except BaseException as e:
print('exception: ' + str(e))
pass
Titles.save()
def scanDLC(id, showErr=True, dlcStatus=None):
id = id.upper()
title = Titles.get(id)
baseDlc = Title.baseDlcId(id)
for i in range(0x1FF):
scanId = format(baseDlc + i, 'X').zfill(16)
if Titles.contains(scanId):
continue
ver = CDNSP.get_version(scanId.lower())
if ver != None:
t = Title()
t.setId(scanId)
Titles.set(scanId, t)
Titles.save()
Print.info('Found new DLC ' + str(title.name) + ' : ' + scanId)
elif showErr:
Print.info('nothing found at ' + scanId + ', ' + str(ver))
if dlcStatus:
dlcStatus.add()
def importRegion(region='US', language='en'):
if not region in Config.regionLanguages(
) or language not in Config.regionLanguages()[region]:
Print.error('Could not locate %s/%s !' % (region, language))
return False
for region2 in Config.regionLanguages():
for language2 in Config.regionLanguages()[region]:
for nsuId, regionTitle in Titles.data(region2, language2).items():
if not regionTitle.id:
continue
title = Titles.get(regionTitle.id, None, None)
title.importFrom(regionTitle, region2, language2)
for region2 in Config.regionLanguages():
for language2 in Config.regionLanguages()[region]:
if language2 != language:
continue
for nsuId, regionTitle in Titles.data(region2, language2).items():
if not regionTitle.id:
continue
title = Titles.get(regionTitle.id, None, None)
title.importFrom(regionTitle, region2, language2)
for nsuId, regionTitle in Titles.data(region, language).items():
if not regionTitle.id:
continue
title = Titles.get(regionTitle.id, None, None)
title.importFrom(regionTitle, region, language)
'''
for k,v in regionTitle.__dict__.items():
if k in ('id', 'version', 'regions', 'languages', 'nsuId', 'key'):
continue
setattr(title, k, v)
title.setId(title.id)
title.setVersion(regionTitle.version)
title.region = region
title.language = language
'''
Titles.save()
def scanBaseThread(baseStatus):
while Config.isRunning:
try:
id = getRandomTitleId()
if Titles.contains(id):
continue
ver = CDNSP.get_version(id.lower())
if ver != None:
Print.info('Found new base ' + id)
t = Title()
t.setId(id)
Titles.set(id, t)
Titles.save()
baseStatus.add()
except BaseException as e:
print('exception: ' + str(e))
def refreshRegions():
for region in Config.regionLanguages():
for language in Config.regionLanguages()[region]:
for i in Titles.data(region, language):
regionTitle = Titles.data(region, language)[i]
if regionTitle.id:
title = Titles.get(regionTitle.id, None, None)
if not hasattr(title, 'regions') or not title.regions:
title.regions = []
if not hasattr(title, 'languages') or not title.languages:
title.languages = []
if not region in title.regions:
title.regions.append(region)
if not language in title.languages:
title.languages.append(language)
Titles.save()
def updateVersions(force = True):
initTitles()
initFiles()
i = 0
for k,t in Titles.items():
if force or t.version == None:
if (t.isDLC or t.isUpdate or Config.download.base) and (not t.isDLC or Config.download.DLC) and (not t.isDemo or Config.download.demo) and (not t.isUpdate or Config.download.update) and (t.key or Config.download.sansTitleKey) and (len(titleWhitelist) == 0 or t.id in titleWhitelist) and t.id not in titleBlacklist:
v = t.lastestVersion(True)
Print.info("%s[%s] v = %s" % (str(t.name), str(t.id), str(v)) )
i = i + 1
if i % 20 == 0:
Titles.save()
for t in list(Titles.data().values()):
if not t.isUpdate and not t.isDLC and t.updateId and t.updateId and not Titles.contains(t.updateId):
u = Title()
u.setId(t.updateId)
if u.lastestVersion():
Titles.set(t.updateId, u)
Print.info("%s[%s] FOUND" % (str(t.name), str(u.id)) )
i = i + 1
if i % 20 == 0:
Titles.save()
Titles.save()
def scrapeTitles(region='US', shop_id=4):
Print.info('Scraping %s' % region)
pageSize = 50
offset = 0
total = 1
c = 0
while offset < total:
url = 'https://bugyo.hac.%s.eshop.nintendo.net/shogun/v1/titles?shop_id=%d&lang=%s&country=%s&sort=new&limit=%d&offset=%d' % (
Config.cdn.environment, shop_id, countryLanguage(region), region,
pageSize, offset)
#print(url)
j = makeJsonRequest(
'GET', url, {}, '%d/%s/%s/titles/index/%d-%d.json' %
(shop_id, countryLanguage(region), region, pageSize, offset))
if not j:
break
total = int(j['total'])
try:
for i in j['contents']:
title = Titles.getNsuid(i['id'])
n = getTitleByNsuid(i['id'], region)
if title:
for x in cdn.Superfly.getAddOns(title.id):
getNsuIds(x, 'aoc', region)
title.parseShogunJson(n)
rt = Title.Title()
rt.setId(title.id)
rt.setRegion(region)
rt.parseShogunJson(n)
Titles.set(title.id, rt, region)
scrapeDlc(i['id'], region)
else:
try:
if n and len(n["applications"]) > 0:
titleId = n["applications"][0]["id"].upper()
for x in cdn.Superfly.getAddOns(titleId):
getNsuIds(x, 'aoc', region)
if titleId:
if Titles.contains(titleId):
title = Titles.get(titleId)
title.setId(titleId)
title.parseShogunJson(n)
#print('existing title found!')
else:
title = Title.Title()
title.setId(titleId)
title.parseShogunJson(n)
Titles.set(titleId, title)
print('added new title %s %s' %
(title.id, title.name))
rt = Title.Title()
rt.setId(titleId)
rt.setRegion(region)
rt.parseShogunJson(n)
Titles.set(titleId, rt, region)
scrapeDlc(i['id'], region)
else:
print('Could not get title json!')
else:
#print('no title id found in json!')
pass
except Exception as e:
#print(str(e))
pass
except Exception as e:
print(str(e))
raise
break
offset = offset + len(j['contents'])
#c = c + 1
#if c % 100 == 0:
# Print.info('.')
# Titles.save()
Titles.save()
def export(file): initTitles() Titles.save(file, ['id', 'rightsId', 'isUpdate', 'isDLC', 'isDemo', 'name', 'version', 'region', 'retailOnly'])
f.close()
if args.create:
Print.info('creating ' + args.create)
nsp = Fs.Nsp(None, None)
nsp.path = args.create
nsp.pack(args.file)
#for filePath in args.file:
# Print.info(filePath)
if args.update_titles:
initTitles()
for url in Config.titleUrls:
updateDb(url)
Titles.loadTxtDatabases()
Titles.save()
if args.submit_keys:
initTitles()
initFiles()
submitKeys()
if args.scrape_languages:
cdn.Shogun.saveLanguages()
exit(0)
if args.refresh_regions:
refreshRegions()
exit(0)
if args.import_region:
def scrapeTitles(region = 'US', shop_id = 4): for language in countryLanguages(region): scrapeLangTitles(region, language, shop_id) Titles.save()
def getUpdateDb(request, response):
for url in Config.titleUrls:
nut.updateDb(url)
Titles.loadTxtDatabases()
Titles.save()
return success(request, response, "Fin")
