def get_aid(self): """ Get video's aid from database :return: """ aid_sql = "SELECT aid FROM aid limit 0, 1;" aid = MySQLDB.select(aid_sql) if aid is None: return url = self.base_url.format(aid[0][0].decode("utf-8")) return url
def mid_select(self): """ Select mid from database :return: """ f = ThreadPoolExecutor(max_workers=1) offset = 0 while True: mid_sql = f"select mid from userinfo where mark=0 limit {offset},10;" mids = MySQLDB.select(mid_sql) if mids is None: break for mid in mids: f.submit(self.get_user_info, mid[0].decode("utf-8")) offset += 10 time.sleep(10) # Based on the request frequency decide how many seconds to sleep
def aid_select(self): """ Select aid from database :return: """ f = ThreadPoolExecutor(max_workers=3) offset = 0 while True: aid_sql = f" select aid from aid where mark=0 limit {offset},10;" aids = MySQLDB.select(aid_sql) if aids is None: break for base_aid in aids: f.submit(self.get_html, base_aid[0].decode("utf-8")) time.sleep(SECONDS) offset += 10 time.sleep( 10) # This is not necessary, depend on the real situation