def saveAll(fileName='titledb/titles.json'):
for region in cdn.regions():
for language in cdn.Shogun.countryLanguages(region):
saveTitlesJson(data(region, language),
'titledb/%s.%s.json' % (region, language))
save(fileName)
def scrapeShogunThreaded():
initTitles()
initFiles()
scrapeThreads = []
numThreads = 8
q = queue.Queue()
for region in cdn.regions():
q.put(region)
for i in range(numThreads):
t = threading.Thread(target=scrapeShogunWorker, args=[q])
t.daemon = True
t.start()
scrapeThreads.append(t)
q.join()
for i in range(numThreads):
q.put(None)
for t in scrapeThreads:
t.join()
Titles.saveAll()
def scrapeShogun():
initTitles()
initFiles()
for region in cdn.regions():
cdn.Shogun.scrapeTitles(region)
Titles.saveAll()
def saveLanguages(fileName = 'titledb/languages.json'):
r = {}
for region in cdn.regions():
r[region] = []
for language in countryLanguages(region):
r[region].append(language)
with open(fileName, 'w') as outfile:
json.dump(r, outfile, indent=4)
def scrapeShogun(force=False, region=None):
initTitles()
initFiles()
if region is None:
for region in cdn.regions():
cdn.Shogun.scrapeTitles(region, force=force)
else:
cdn.Shogun.scrapeTitles(region, force=force)
Titles.saveAll()
def save(fileName='titledb/titles.json'):
for region in cdn.regions():
saveTitlesJson(data(region), 'titledb/%s.json' % region)
confLock.acquire()
try:
j = {}
for i, k in titles.items():
if not k.id or k.id == '0000000000000000':
continue
if k.description:
k.description = k.description.strip()
j[k.id] = k.__dict__
with open(fileName, 'w') as outfile:
json.dump(j, outfile, indent=4)
except:
confLock.release()
raise
confLock.release()
def scrapeShogunThreaded(force=False):
if not hasCdn:
return
initTitles()
initFiles()
scrapeThreads = []
numThreads = 8
if Config.reverse:
q = queue.LifoQueue()
else:
q = queue.Queue()
for region in cdn.regions():
q.put(region)
for i in range(numThreads):
t = threading.Thread(target=scrapeShogunWorker, args=[q, force])
t.daemon = True
t.start()
scrapeThreads.append(t)
Print.info('joining shogun queue')
q.join()
for i in range(numThreads):
q.put(None)
i = 0
for t in scrapeThreads:
i += 1
t.join()
Print.info('joined thread %d of %d' % (i, len(scrapeThreads)))
# q.join()
Print.info('saving titles')
Titles.saveAll()
Print.info('titles saved')
def load():
global regionTitles
for region in cdn.regions():
filePath = 'titledb/%s.json' % region
if os.path.isfile(filePath):
regionTitles[region] = loadTitlesJson(filePath)
else:
regionTitles[region] = {}
confLock.acquire()
global titles
if os.path.isfile("titledb/titles.json"):
timestamp = time.clock()
with open('titledb/titles.json', encoding="utf-8-sig") as f:
for i, k in json.loads(f.read()).items():
titles[i] = Title.Title()
titles[i].__dict__ = k
Print.info('loaded titledb/titles.json in ' +
str(time.clock() - timestamp) + ' seconds')
if os.path.isfile("titles.txt"):
loadTitleFile('titles.txt', True)
try:
files = [
f for f in os.listdir(Config.paths.titleDatabase)
if f.endswith('.txt')
]
files.sort()
for file in files:
loadTitleFile(Config.paths.titleDatabase + '/' + file, False)
except BaseException as e:
Print.error('title load error: ' + str(e))
confLock.release()
