def load_mid(self, limit):
'''
:param limit: limit是要查询的是哪个有效的url种类
:return: 返回的是dict
'''
db = MySql()
sql = f"select mid from baida_up where {limit} = 1"
res = db.fetch_all(sql)
return res
def load_mid_without_limit(self):
'''
:param limit: limit是要查询的是哪个有效的url种类
:return: 返回的是dict
'''
db = MySql()
sql = f"select mid from baida_up"
res = db.fetch_all(sql)
return res
def load_url_3(self):
sql = "select mid,url from getbvlist where valid=1"
db = MySql()
res = db.fetch_all(sql)
info = []
for i in res:
pair = []
pair.append(i[0])
pair.append(i[1])
info.append(pair)
return info
def load_url_5(self):
sql = "select av,url_2 from video_url where valid_2 =1"
db = MySql()
res = db.fetch_all(sql)
info = []
for i in res:
pair = []
pair.append(i[0])
pair.append(i[1])
info.append(pair)
return info
def load_url_2(self):
sql = "select mid,url_2 from baida_up where url_2_valid=1"
db = MySql()
res = db.fetch_all(sql)
info = []
for i in res:
pair = []
pair.append(i[0])
pair.append(i[1])
info.append(pair)
print(info)
return info
def execute_many(self, name): #将url初始化进入数据库
db = MySql()
sql = []
if name == "build_urls_1":
sql = self.build_urls_1()
elif name == "build_urls_2":
sql = self.build_urls_2()
elif name == "build_url4_and_url5":
sql = self.build_url4_and_url5()
else:
assert "excute_many name is wrong"
for i in sql:
print(i)
db.execute(i)
print("ok")
def load_url_1(self):
'''
:return: mid,url_1
'''
sql = "select mid,url_1 from baida_up where url_1_valid=1 "
db = MySql()
res = db.fetch_all(sql)
info = []
for i in res:
pair = []
pair.append(i[0])
pair.append(i[1])
info.append(pair)
return info
def deal_url_2_res(self, res, mid):
info = {}
print("res", res)
for i, j in res["data"].items():
if i in self.need:
if j == "":
j = " "
info[i] = j
sql2 = f'''update upbasic set following={info["following"]},black={info["black"]},follower={info["follower"]} where mid = {mid}'''
sql3 = f'''update baida_up set url_2_valid = 0 where mid = {mid}'''
db = MySql()
print(sql2)
db.execute(sql2)
db.execute(sql3)
print("ok")
def input_url3(self):
loads = self.load_mid_without_limit()
db = MySql()
for i in loads:
pn = 1
mid = i[0]
sql1 = f'''select sum from getbvlist where mid={mid}'''
res = int(db.fetch_one(sql1)[0])
while (res > 0):
res = res - 100
pn += 1
url = self.format_url_3(mid, pn)
sql = f'''Insert Into getbvlist values({mid},"{url}",1,{res})'''
print(sql)
db.execute(sql)
print(mid)
def deal_url_1_res(self, res, mid):
info = {}
for i, j in res["data"].items():
if i in self.need:
if j == "":
j = " "
info[i] = j
sql2 = f'''insert into upbasic values({info["mid"]},"{info["name"]}","{info["sex"]}","null",{info["level"]},"{info["birthday"]}",{info["coins"]},null,null,null)'''
sql1 = f'''select mid from upbasic where mid = {mid}'''
sql3 = f'''update baida_up set url_1_valid = 0 where mid = {mid}'''
db = MySql()
call = db.fetch_one(sql1)
print(call)
if call == None:
print(sql2)
db.execute(sql2)
db.execute(sql3)
print("ok")
def add_ip(self):#将ip添入ip_pool
'''
将代理ip添加入ip_pool
:return:如果成功，返回true，否则，False
'''
print("正在添加~~~~~~")
today = datetime.date.today()
sql = f'''select ip from ip_pool where valid=1 and date = "{today}"'''
db = MySql()
res = db.fetch_all(sql)
print(type(res))
if res != tuple():
for i in res:
self.ip_pool.append(i[0])
print("添加成功")
return True
print("添加失败，数据库中没有合适的值")
return False
def deal_url_5_res(self, dict_json, av, url):
db = MySql()
for i in dict_json["data"]:
title = i[title]
sql = f'''Update bvlist set title="{title} where av = {av}"'''
db.execute(sql)
sql2 = f'''Update video_url valid_2=0 where av={av}'''
db.execute(sql2)
def deal_url_3_res(self, dict_json, mid, url):
videolist = []
moreinfo = {}
for i in dict_json["data"]["list"]["vlist"]:
videolist.append(str(i["aid"]))
moreinfo[str(i["aid"])] = {}
for keyj, valuej in i.items():
if keyj in self.videoneed:
moreinfo[str(i["aid"])][keyj] = valuej
db = MySql()
for i in videolist:
sql_check = f'''select mid from bvlist where av = "{i}" '''
res = db.execute(sql_check)
print("res", res)
if res == 0:
sql = f'''Insert into bvlist values({mid},null,"{i}",1,{moreinfo[i]["typeid"]},{moreinfo[i]["created"]},"{moreinfo[i]["length"]}",{moreinfo[i]["is_union_video"]})''' #将信息插入bvlist
print(sql)
db.execute(sql)
sql2 = f'''update getbvlist set valid=0 where url="{url}"'''
db.execute(sql2)
print("videolist", videolist)
print("moreinfo", moreinfo)
def deal_url_4_res(self, dict_json, av, url):
print(av)
if av == 0 or av == 1299:
return
db = MySql()
i = dict_json["data"]
view = i["view"]
if isinstance(view, str): #用来处理视频不见了的状况
view = -1
danmaku = i["danmaku"]
reply = i["reply"]
favorite = i["favorite"]
coin = i["coin"]
share = i["share"]
his_rank = i["his_rank"]
like = i["like"]
sql = f'''Update bvlist set view={view},danmaku={danmaku},reply={reply},favorite={favorite},coin={coin},share={share},his_rank={his_rank},likes={like} where av={av}'''
print(sql)
db.execute(sql)
sql2 = f'''Update video_url set valid_1=0 where av={av}'''
db.execute(sql2)
def load_av(self):
db = MySql()
sql = f'''select av from bvlist'''
res = db.fetch_all(sql)
return res
def update_valid(self,ip):#将valid变为0
sql = f'''update ip_pool set valid = 0 where ip="{ip}"'''
db = MySql()
res = db.execute(sql)
print("delete ok")
def delete_today(self):#将今天的数据全部删除
sql = "Delete from ip_pool where 1=1"
db = MySql()
res = db.execute(sql)
print("delete all ok")
def load_url_1(self):
db = MySql()
sql = "select url_1 from baida_up where url_1_valid=1"
res = db.fetch_all(sql)
for i in res:
self.url_item.append(i[0])
def ip_store(self,ip):
db = MySql()
sql = f'''insert into ip_pool values(null,'{ip}',NOW()，1)'''
print(sql)
db.execute(sql)
