def saveAll(fileName='titledb/titles.json'): for region in cdn.regions(): for language in cdn.Shogun.countryLanguages(region): saveTitlesJson(data(region, language), 'titledb/%s.%s.json' % (region, language)) save(fileName)
def scrapeShogunThreaded(): initTitles() initFiles() scrapeThreads = [] numThreads = 8 q = queue.Queue() for region in cdn.regions(): q.put(region) for i in range(numThreads): t = threading.Thread(target=scrapeShogunWorker, args=[q]) t.daemon = True t.start() scrapeThreads.append(t) q.join() for i in range(numThreads): q.put(None) for t in scrapeThreads: t.join() Titles.saveAll()
def scrapeShogun(): initTitles() initFiles() for region in cdn.regions(): cdn.Shogun.scrapeTitles(region) Titles.saveAll()
def saveLanguages(fileName = 'titledb/languages.json'): r = {} for region in cdn.regions(): r[region] = [] for language in countryLanguages(region): r[region].append(language) with open(fileName, 'w') as outfile: json.dump(r, outfile, indent=4)
def scrapeShogun(force=False, region=None): initTitles() initFiles() if region is None: for region in cdn.regions(): cdn.Shogun.scrapeTitles(region, force=force) else: cdn.Shogun.scrapeTitles(region, force=force) Titles.saveAll()
def save(fileName='titledb/titles.json'): for region in cdn.regions(): saveTitlesJson(data(region), 'titledb/%s.json' % region) confLock.acquire() try: j = {} for i, k in titles.items(): if not k.id or k.id == '0000000000000000': continue if k.description: k.description = k.description.strip() j[k.id] = k.__dict__ with open(fileName, 'w') as outfile: json.dump(j, outfile, indent=4) except: confLock.release() raise confLock.release()
def scrapeShogunThreaded(force=False): if not hasCdn: return initTitles() initFiles() scrapeThreads = [] numThreads = 8 if Config.reverse: q = queue.LifoQueue() else: q = queue.Queue() for region in cdn.regions(): q.put(region) for i in range(numThreads): t = threading.Thread(target=scrapeShogunWorker, args=[q, force]) t.daemon = True t.start() scrapeThreads.append(t) Print.info('joining shogun queue') q.join() for i in range(numThreads): q.put(None) i = 0 for t in scrapeThreads: i += 1 t.join() Print.info('joined thread %d of %d' % (i, len(scrapeThreads))) # q.join() Print.info('saving titles') Titles.saveAll() Print.info('titles saved')
def load(): global regionTitles for region in cdn.regions(): filePath = 'titledb/%s.json' % region if os.path.isfile(filePath): regionTitles[region] = loadTitlesJson(filePath) else: regionTitles[region] = {} confLock.acquire() global titles if os.path.isfile("titledb/titles.json"): timestamp = time.clock() with open('titledb/titles.json', encoding="utf-8-sig") as f: for i, k in json.loads(f.read()).items(): titles[i] = Title.Title() titles[i].__dict__ = k Print.info('loaded titledb/titles.json in ' + str(time.clock() - timestamp) + ' seconds') if os.path.isfile("titles.txt"): loadTitleFile('titles.txt', True) try: files = [ f for f in os.listdir(Config.paths.titleDatabase) if f.endswith('.txt') ] files.sort() for file in files: loadTitleFile(Config.paths.titleDatabase + '/' + file, False) except BaseException as e: Print.error('title load error: ' + str(e)) confLock.release()