def refresh_data(self, loop): self.db_refreshing = True log.info('refreshing data...') semaphore = asyncio.Semaphore(5) f = asyncio.wait([last_page(semaphore=semaphore, server=Server.BEFORE_BIG_BANG, section=Section.MONSTER, subsection=lvl) for lvl in MonsterLevelType]) finished, pending = loop.run_until_complete(f) cats_pages = [itertools.product((category,), range(0, page_nr + 1)) for page_nr, category in map(lambda task: task.result(), finished)] cats_pages = tuple(itertools.chain.from_iterable(cats_pages)) log.debug(cats_pages) f = asyncio.wait([scrape(semaphore=semaphore, server=Server.BEFORE_BIG_BANG, section=Section.MONSTER, subsection=lvl, page=page, model=Monster) for lvl, page in cats_pages]) finished, pending = loop.run_until_complete(f) if __name__ == '__main__': loop.close() if Weapon.select().count() > 0: Weapon.delete().execute() if Monster.select().count() > 0: Monster.delete().execute() with self.db.atomic(): for result, category in [task.result() for task in finished]: model = type(category).related_model() for i in range(0, len(result), self.max_bulk_rows): top = i + self.max_bulk_rows model.insert_many(result[i:top]).execute() if len(result) == 0: log.warn('empty result set for {} category'.format(category)) self.db_refreshing = False log.info('refreshing data... done')
def monsters_by_name(self, monster_name: str, exact_match: bool=False): if self.db_refreshing: raise ValueError('Database refresh in progress') keyw = monster_name if exact_match else '%{}%'.format(monster_name) q = Monster.select().where(Monster.name ** keyw) return q.execute()