def downloadAll(wait = True): global activeDownloads global status try: for k,t in Titles.items(): if t.isUpdateAvailable() and not t.retailOnly and (t.isDLC or t.isUpdate or Config.download.base) and (not t.isDLC or Config.download.DLC) and (not t.isDemo or Config.download.demo) and (not t.isUpdate or Config.download.update) and (t.key or Config.download.sansTitleKey) and (len(titleWhitelist) == 0 or t.id in titleWhitelist) and t.id not in titleBlacklist: if not t.id or t.id == '0' * 16 or (t.isUpdate and t.lastestVersion() in [None, '0']): #Print.warning('no valid id? ' + str(t.path)) continue if not t.lastestVersion(): Print.info('Could not get version for ' + str(t.name)) continue Titles.queue.add(t.id) Titles.save() status = Status.create(Titles.queue.size(), 'Total Download') startDownloadThreads() while wait and (not Titles.queue.empty() or sum(activeDownloads) > 0): time.sleep(1) except KeyboardInterrupt: pass except BaseException as e: Print.error(str(e)) if status: status.close()
def scanLatestTitleUpdates(): initTitles() initFiles() for k, i in CDNSP.get_versionUpdates().items(): id = str(k).upper() version = str(i) if not Titles.contains(id): if len(id) != 16: Print.info('invalid title id: ' + id) continue continue t = Title() t.setId(id) Titles.set(id, t) Print.info('Found new title id: ' + str(id)) t = Titles.get(id) if str(t.version) != str(version): Print.info('new version detected for %s[%s] v%s' % (t.name or '', t.id or ('0' * 16), str(version))) t.setVersion(version, True) Titles.save()
def importRegion(region='US', language='en'): if not region in Config.regionLanguages( ) or language not in Config.regionLanguages()[region]: Print.error('Could not locate %s/%s !' % (region, language)) return False for region2 in Config.regionLanguages(): for language2 in Config.regionLanguages()[region2]: for nsuId, regionTitle in Titles.data(region2, language2).items(): if not regionTitle.id: continue title = Titles.get(regionTitle.id, None, None) title.importFrom(regionTitle, region2, language2) for region2 in Config.regionLanguages(): for language2 in Config.regionLanguages()[region2]: if language2 != language: continue for nsuId, regionTitle in Titles.data(region2, language2).items(): if not regionTitle.id: continue title = Titles.get(regionTitle.id, None, None) title.importFrom(regionTitle, region2, language2) for nsuId, regionTitle in Titles.data(region, language).items(): if not regionTitle.id: continue title = Titles.get(regionTitle.id, None, None) title.importFrom(regionTitle, region, language) Titles.loadTxtDatabases() Titles.save()
def genTinfoilTitles(): initTitles() initFiles() for region, languages in Config.regionLanguages().items(): for language in languages: importRegion(region, language) Titles.save('titledb/titles.%s.%s.json' % (region, language))
def scan(): global hasScanned if hasScanned: return hasScanned = True initTitles() initFiles() Nsps.scan(Config.paths.scan) Titles.save()
def genTinfoilTitles(): initTitles() initFiles() for region, languages in Config.regionLanguages().items(): for language in languages: importRegion(region, language) Titles.save('titledb/titles.%s.%s.json' % (region, language)) #Print.info('%s - %s' % (region, language)) scanLatestTitleUpdates() export('titledb/versions.txt', ['id', 'version'])
def refresh(): initTitles() initFiles() for k, f in Nsps.files.items(): try: f.open() f.readMeta() f.close() except: raise pass Titles.save()
def scan(): global hasScanned #if hasScanned: # return hasScanned = True initTitles() initFiles() refreshRegions() importRegion(Config.region, Config.language) r = Nsps.scan(Config.paths.scan) Titles.save() return r
def refresh(titleRightsOnly = False): initTitles() initFiles() for k, f in Nsps.files.items(): try: if titleRightsOnly: title = Titles.get(f.titleId) if title and title.rightsId: continue print(f.path) f.open() f.readMeta() f.close() except BaseException as e: print('exception: ' + str(e)) pass Titles.save()
def scanDLC(id, showErr=True, dlcStatus=None): id = id.upper() title = Titles.get(id) baseDlc = Title.baseDlcId(id) for i in range(0x1FF): scanId = format(baseDlc + i, 'X').zfill(16) if Titles.contains(scanId): continue ver = CDNSP.get_version(scanId.lower()) if ver != None: t = Title() t.setId(scanId) Titles.set(scanId, t) Titles.save() Print.info('Found new DLC ' + str(title.name) + ' : ' + scanId) elif showErr: Print.info('nothing found at ' + scanId + ', ' + str(ver)) if dlcStatus: dlcStatus.add()
def importRegion(region='US', language='en'): if not region in Config.regionLanguages( ) or language not in Config.regionLanguages()[region]: Print.error('Could not locate %s/%s !' % (region, language)) return False for region2 in Config.regionLanguages(): for language2 in Config.regionLanguages()[region]: for nsuId, regionTitle in Titles.data(region2, language2).items(): if not regionTitle.id: continue title = Titles.get(regionTitle.id, None, None) title.importFrom(regionTitle, region2, language2) for region2 in Config.regionLanguages(): for language2 in Config.regionLanguages()[region]: if language2 != language: continue for nsuId, regionTitle in Titles.data(region2, language2).items(): if not regionTitle.id: continue title = Titles.get(regionTitle.id, None, None) title.importFrom(regionTitle, region2, language2) for nsuId, regionTitle in Titles.data(region, language).items(): if not regionTitle.id: continue title = Titles.get(regionTitle.id, None, None) title.importFrom(regionTitle, region, language) ''' for k,v in regionTitle.__dict__.items(): if k in ('id', 'version', 'regions', 'languages', 'nsuId', 'key'): continue setattr(title, k, v) title.setId(title.id) title.setVersion(regionTitle.version) title.region = region title.language = language ''' Titles.save()
def scanBaseThread(baseStatus): while Config.isRunning: try: id = getRandomTitleId() if Titles.contains(id): continue ver = CDNSP.get_version(id.lower()) if ver != None: Print.info('Found new base ' + id) t = Title() t.setId(id) Titles.set(id, t) Titles.save() baseStatus.add() except BaseException as e: print('exception: ' + str(e))
def refreshRegions(): for region in Config.regionLanguages(): for language in Config.regionLanguages()[region]: for i in Titles.data(region, language): regionTitle = Titles.data(region, language)[i] if regionTitle.id: title = Titles.get(regionTitle.id, None, None) if not hasattr(title, 'regions') or not title.regions: title.regions = [] if not hasattr(title, 'languages') or not title.languages: title.languages = [] if not region in title.regions: title.regions.append(region) if not language in title.languages: title.languages.append(language) Titles.save()
def updateVersions(force = True): initTitles() initFiles() i = 0 for k,t in Titles.items(): if force or t.version == None: if (t.isDLC or t.isUpdate or Config.download.base) and (not t.isDLC or Config.download.DLC) and (not t.isDemo or Config.download.demo) and (not t.isUpdate or Config.download.update) and (t.key or Config.download.sansTitleKey) and (len(titleWhitelist) == 0 or t.id in titleWhitelist) and t.id not in titleBlacklist: v = t.lastestVersion(True) Print.info("%s[%s] v = %s" % (str(t.name), str(t.id), str(v)) ) i = i + 1 if i % 20 == 0: Titles.save() for t in list(Titles.data().values()): if not t.isUpdate and not t.isDLC and t.updateId and t.updateId and not Titles.contains(t.updateId): u = Title() u.setId(t.updateId) if u.lastestVersion(): Titles.set(t.updateId, u) Print.info("%s[%s] FOUND" % (str(t.name), str(u.id)) ) i = i + 1 if i % 20 == 0: Titles.save() Titles.save()
def scrapeTitles(region='US', shop_id=4): Print.info('Scraping %s' % region) pageSize = 50 offset = 0 total = 1 c = 0 while offset < total: url = 'https://bugyo.hac.%s.eshop.nintendo.net/shogun/v1/titles?shop_id=%d&lang=%s&country=%s&sort=new&limit=%d&offset=%d' % ( Config.cdn.environment, shop_id, countryLanguage(region), region, pageSize, offset) #print(url) j = makeJsonRequest( 'GET', url, {}, '%d/%s/%s/titles/index/%d-%d.json' % (shop_id, countryLanguage(region), region, pageSize, offset)) if not j: break total = int(j['total']) try: for i in j['contents']: title = Titles.getNsuid(i['id']) n = getTitleByNsuid(i['id'], region) if title: for x in cdn.Superfly.getAddOns(title.id): getNsuIds(x, 'aoc', region) title.parseShogunJson(n) rt = Title.Title() rt.setId(title.id) rt.setRegion(region) rt.parseShogunJson(n) Titles.set(title.id, rt, region) scrapeDlc(i['id'], region) else: try: if n and len(n["applications"]) > 0: titleId = n["applications"][0]["id"].upper() for x in cdn.Superfly.getAddOns(titleId): getNsuIds(x, 'aoc', region) if titleId: if Titles.contains(titleId): title = Titles.get(titleId) title.setId(titleId) title.parseShogunJson(n) #print('existing title found!') else: title = Title.Title() title.setId(titleId) title.parseShogunJson(n) Titles.set(titleId, title) print('added new title %s %s' % (title.id, title.name)) rt = Title.Title() rt.setId(titleId) rt.setRegion(region) rt.parseShogunJson(n) Titles.set(titleId, rt, region) scrapeDlc(i['id'], region) else: print('Could not get title json!') else: #print('no title id found in json!') pass except Exception as e: #print(str(e)) pass except Exception as e: print(str(e)) raise break offset = offset + len(j['contents']) #c = c + 1 #if c % 100 == 0: # Print.info('.') # Titles.save() Titles.save()
def export(file): initTitles() Titles.save(file, ['id', 'rightsId', 'isUpdate', 'isDLC', 'isDemo', 'name', 'version', 'region', 'retailOnly'])
f.close() if args.create: Print.info('creating ' + args.create) nsp = Fs.Nsp(None, None) nsp.path = args.create nsp.pack(args.file) #for filePath in args.file: # Print.info(filePath) if args.update_titles: initTitles() for url in Config.titleUrls: updateDb(url) Titles.loadTxtDatabases() Titles.save() if args.submit_keys: initTitles() initFiles() submitKeys() if args.scrape_languages: cdn.Shogun.saveLanguages() exit(0) if args.refresh_regions: refreshRegions() exit(0) if args.import_region:
def scrapeTitles(region = 'US', shop_id = 4): for language in countryLanguages(region): scrapeLangTitles(region, language, shop_id) Titles.save()
def getUpdateDb(request, response): for url in Config.titleUrls: nut.updateDb(url) Titles.loadTxtDatabases() Titles.save() return success(request, response, "Fin")