async def timed_crawler(self, url, freq, ws_id): redis_conn = redis.Redis(connection_pool=self.redis) while True: ws_freq_check = redis_conn.get(ws_id) if not ws_freq_check or not int(ws_freq_check) >= 0: logging.info('got shutdown signal {}'.format(str(ws_id))) break crawler = Crawler([url], loop=self.loop, db_rpc=DbRpcClient()) await crawler.crawl() crawler.close() await asyncio.sleep(int(freq) * 60)
async def one_time_crawler(self, url, freq, ws_id): crawler = Crawler([url], loop=self.loop, db_rpc=DbRpcClient()) await crawler.crawl() crawler.close()