async def timed_crawler(self, url, freq, ws_id):
redis_conn = redis.Redis(connection_pool=self.redis)
while True:
ws_freq_check = redis_conn.get(ws_id)
if not ws_freq_check or not int(ws_freq_check) >= 0:
logging.info('got shutdown signal {}'.format(str(ws_id)))
break
crawler = Crawler([url], loop=self.loop, db_rpc=DbRpcClient())
await crawler.crawl()
crawler.close()
await asyncio.sleep(int(freq) * 60)
async def one_time_crawler(self, url, freq, ws_id):
crawler = Crawler([url], loop=self.loop, db_rpc=DbRpcClient())
await crawler.crawl()
crawler.close()