def refresh_data(self, loop):
self.db_refreshing = True
log.info('refreshing data...')
semaphore = asyncio.Semaphore(5)
f = asyncio.wait([last_page(semaphore=semaphore,
server=Server.BEFORE_BIG_BANG,
section=Section.MONSTER,
subsection=lvl)
for lvl in MonsterLevelType])
finished, pending = loop.run_until_complete(f)
cats_pages = [itertools.product((category,), range(0, page_nr + 1))
for page_nr, category in
map(lambda task: task.result(), finished)]
cats_pages = tuple(itertools.chain.from_iterable(cats_pages))
log.debug(cats_pages)
f = asyncio.wait([scrape(semaphore=semaphore,
server=Server.BEFORE_BIG_BANG,
section=Section.MONSTER,
subsection=lvl,
page=page,
model=Monster)
for lvl, page in cats_pages])
finished, pending = loop.run_until_complete(f)
if __name__ == '__main__':
loop.close()
if Weapon.select().count() > 0:
Weapon.delete().execute()
if Monster.select().count() > 0:
Monster.delete().execute()
with self.db.atomic():
for result, category in [task.result() for task in finished]:
model = type(category).related_model()
for i in range(0, len(result), self.max_bulk_rows):
top = i + self.max_bulk_rows
model.insert_many(result[i:top]).execute()
if len(result) == 0:
log.warn('empty result set for {} category'.format(category))
self.db_refreshing = False
log.info('refreshing data... done')
def monsters_by_name(self, monster_name: str, exact_match: bool=False):
if self.db_refreshing:
raise ValueError('Database refresh in progress')
keyw = monster_name if exact_match else '%{}%'.format(monster_name)
q = Monster.select().where(Monster.name ** keyw)
return q.execute()
