def insert_mid(mid_list):
"""
Save user's mid to userinfo table
:return:
"""
user_sql = "insert into userinfo(mid) values({});"
MySQLDB.insert_many(user_sql, mid_list)
def mark_mid(self, base_mid):
"""
Mark mid been used
:param base_mid:
:return:
"""
mark_sql = f"UPDATE userinfo SET MARK=1 WHERE mid={base_mid}"
MySQLDB.update(mark_sql)
def update_user_info(self, user_info):
"""
Update user's information to database
:param user_info:
:return:
"""
user_sql = "UPDATE userinfo SET name=%s, sex=%s, face=%s, regtime=%s, sign=%s, birthday=%s, place=%s, current_level=%s, fans=%s, friends=%s, attention=%s, archive_count=%s, article_count=%s, mark=1 where mid=%s;"
MySQLDB.update(user_sql, user_info)
def aid_insert(aids, base_aid):
"""
Insert video's aid to aid table and mark the aid been used
:return:
"""
aid_sql = "INSERT INTO aid(aid) VALUES ({});"
MySQLDB.insert_many(aid_sql, aids)
mark_sql = "UPDATE aid SET mark=1 where aid={};".format(base_aid)
MySQLDB.update(mark_sql)
def get_aid(self):
"""
Get video's aid from database
:return:
"""
aid_sql = "SELECT aid FROM aid limit 0, 1;"
aid = MySQLDB.select(aid_sql)
if aid is None:
return
url = self.base_url.format(aid[0][0].decode("utf-8"))
return url
def mid_select(self):
"""
Select mid from database
:return:
"""
f = ThreadPoolExecutor(max_workers=1)
offset = 0
while True:
mid_sql = f"select mid from userinfo where mark=0 limit {offset},10;"
mids = MySQLDB.select(mid_sql)
if mids is None:
break
for mid in mids:
f.submit(self.get_user_info, mid[0].decode("utf-8"))
offset += 10
time.sleep(10) # Based on the request frequency decide how many seconds to sleep
def aid_select(self):
"""
Select aid from database
:return:
"""
f = ThreadPoolExecutor(max_workers=3)
offset = 0
while True:
aid_sql = f" select aid from aid where mark=0 limit {offset},10;"
aids = MySQLDB.select(aid_sql)
if aids is None:
break
for base_aid in aids:
f.submit(self.get_html, base_aid[0].decode("utf-8"))
time.sleep(SECONDS)
offset += 10
time.sleep(
10) # This is not necessary, depend on the real situation