def run():
user = tool.AccountManager("Anki")
cookie = user.getCookies()
db = tool.getDB()
rs = db.execute(
"select distinct bvid from data where zht=false or zhs=false or en=false"
).fetchall()
api = "https://api.bilibili.com/x/player/pagelist?bvid="
s = tool.Session()
s.cookies.update(cookie)
for bvid in rs:
bvid = bvid[0]
pages = s.get(api + bvid).json()
if pages["code"] != 0:
# db.execute("delete from data where bvid=?", (bvid, ))
continue
pages = pages["data"]
vid = db.execute("select vid,zht,zhs,en from data where bvid=?",
(bvid, )).fetchone()
if len(pages) == 1: # 不分P视频
send((bvid, pages[0]["cid"]) + vid, cookie)
continue
for i in pages: # 分P视频
title = i["part"]
vid = db.execute("select vid,zht,zhs,en from data where title=?",
(title, )).fetchone()
if vid is None or len(vid) == 0:
continue
send((bvid, i["cid"]) + vid, cookie)
db.close()
Subtitle.fix_sub(cookie=cookie)
def send_subtitle(bvid, lan, cid, cookie, fix=False, vid=None, add=None):
logger = tool.getLogger()
_api = "https://api.bilibili.com/x/v2/dm/subtitle/draft/save"
csrf = cookie["bili_jct"]
s = tool.Session()
s.cookies.update(cookie)
if not fix:
sou = get_sub(vid, lan)
else:
sou = add
if sou is None:
return False
send_data = {
"type": 1,
"oid": cid,
"bvid": bvid,
"lan": lan,
"data": sou,
"submit": "true",
"sign": "false",
"csrf": csrf
}
# print(re.findall("bili_jct=(.*?);", cookie)[0])
# print(parse.urlencode(send_data))
# print(json.dumps(send_data, ensure_ascii=False))
# s.verify = False
_res = s.post(url=_api, data=send_data).json()
if _res["code"] != 0:
logger.error(str(bvid) + json.dumps(_res))
return False
logger.info(
f"subtitle success BV[{bvid}], Lan[{lan}], fix[{fix}], cid[{cid}]")
return True
def getVideoUrl(self):
s = tool.Session()
s.headers.update({
"Sec-Fetch-Dest": "empty",
"X-Requested-With": "XMLHttpRequest",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36",
"Content-Type": "application/json;charset=UTF-8",
"Accept": "*/*",
"Origin": "https://www.y2b.xyz",
"Sec-Fetch-Site": "same-origin",
"Sec-Fetch-Mode": "cors",
"Referer": "https://www.y2b.xyz/",
"Accept-Language": "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7,zh-TW;q=0.6",
})
raw = s.get("https://www.y2b.xyz/", useProxy=True,
wantStatusCode=200).text
csrf = re.findall('csrf_token = "([^"]+)', raw)[0]
s.headers.update({
"X-XSRF-TOKEN": parse.unquote(s.cookies.get_dict()["XSRF-TOKEN"]),
"X-CSRF-TOKEN": csrf
})
rs = s.post("https://www.y2b.xyz/analysis",
json={"url": f"https://www.youtube.com/watch?v={self.vid}",
"channel": "one"},
useProxy=True,
wantStatusCode=200).json()
s.close()
return rs
def uploadWithOldBvid(cookie: dict, uploadInfo: dict, videoPath: str):
logger = tool.getLogger()
enableParallel = uploadInfo.get("enableParallel", True)
success, upos_uri = uploadFile(cookie,
videoPath,
enableParallel=enableParallel)
if not success:
return False, "", ""
s = tool.Session()
s.cookies.update(cookie)
url = f"https://member.bilibili.com/x/vu/web/edit?csrf={cookie['bili_jct']}"
# s.headers.pop("X-Upos-Auth")
_rs = s.get(
f"https://member.bilibili.com/x/web/archive/view?bvid={uploadInfo['bvid']}"
).json()["data"]
# logger.debug(json.dumps(_rs["videos"]))
videos = []
for i in _rs["videos"]:
if len(i['reject_reason']) > 0: # 判断视频是否有错误，比如撞车、解码错误、违法违规等
logger.debug("{}-{}:{}".format(i["aid"], i["cid"],
i["reject_reason"]))
continue
videos.append({"filename": i["filename"], "title": i["title"]})
videos.append({
"filename":
upos_uri.split(".")[0],
"title":
uploadInfo["title"][0:min(79, len(uploadInfo["title"]))],
"desc":
uploadInfo["id"]
})
send_data = {
"copyright": 2,
"videos": videos,
"source": _rs["archive"]["source"],
"tid": _rs["archive"]["tid"],
"cover": _rs["archive"]["cover"].split(":")[-1],
"title": _rs["archive"]["title"],
"tag": _rs["archive"]["tag"],
"desc_format_id": 0,
"desc": _rs["archive"]["desc"],
"dynamic": _rs["archive"]["dynamic"],
"subtitle": {
"open": 0,
"lan": ""
},
"bvid": uploadInfo["bvid"],
"handle_staff": False,
}
logger.debug(json.dumps(send_data))
# s.headers.update({"Content-Type": "application/json;charset=UTF-8"})
res = s.post(url=url, json=send_data).text
logger.debug(res)
return True, res, upos_uri
def uploadWithNewBvid(cookie: dict, uploadInfo: dict, videoPath: str):
logger = tool.getLogger()
enableParallel = uploadInfo.get("enableParallel", False)
success, upos_uri = uploadFile(
cookie, videoPath, enableParallel=enableParallel)
if not success:
return False, "", ""
s = tool.Session()
s.cookies.update(cookie)
csrf = cookie["bili_jct"]
def cover(csrf, uploadInfo):
vid = uploadInfo["id"]
__url = "https://member.bilibili.com/x/vu/web/cover/up"
__imgURL = f"https://i1.ytimg.com/vi/{vid}/maxresdefault.jpg"
__imgURL2 = f"https://i1.ytimg.com/vi/{vid}/hqdefault.jpg"
__rs = s.get(__imgURL, useProxy=True, wantStatusCode=200)
if __rs is None:
__rs = s.get(__imgURL2, useProxy=True, wantStatusCode=200)
__send = {"cover": "data:image/jpeg;base64," +
base64.b64encode(__rs.content).decode(),
"csrf": csrf
}
__res = s.post(url=__url, data=__send).json()
return __res["data"]["url"].replace("http:", "").replace("https:", "")
url = "https://member.bilibili.com/x/vu/web/add?csrf=" + csrf
# s.headers.pop("X-Upos-Auth")
_data = s.get("https://member.bilibili.com/x/geetest/pre/add").text
logger.debug(_data)
send_data = {"copyright": 2,
"videos": [{"filename": upos_uri.split(".")[0],
"title": uploadInfo["title"],
"desc": ""}],
"source": "https://www.youtube.com/watch?v=" + uploadInfo["id"],
"tid": int(uploadInfo["tid"]),
"cover": cover(csrf, uploadInfo),
"title": uploadInfo["ptitle"],
"tag": ','.join(uploadInfo["tags"]),
"desc_format_id": 0,
"desc": uploadInfo["desc"],
"dynamic": "#" + "##".join(uploadInfo["tags"]) + "#",
"subtitle": {
"open": 0,
"lan": ""}
}
logger.debug(json.dumps(send_data))
# s.headers.update({"Content-Type": "application/json;charset=UTF-8"})
res = s.post(url=url, json=send_data).text
logger.debug(res)
s.close()
return True, res, upos_uri
def getVideoUrlByYoutubeApi(self):
s = tool.Session()
s.headers.update({
"X-Requested-With": "XMLHttpRequest",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36",
"Accept": "*/*",
"Origin": "https://www.youtube.com/",
"Referer": "https://www.youtube.com/",
"Accept-Language": "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7,zh-TW;q=0.6",
})
url = f"https://www.youtube.com/get_video_info?video_id={self.vid}"
rs = s.get(url, useProxy=True).text
a = parse.parse_qs(rs.text)
a = json.loads(a['player_response'][0])['streamingData']['adaptiveFormats']
s.close()
def fix_sub(cookie):
logger = tool.getLogger()
csrf = cookie["bili_jct"]
s = tool.Session()
s.cookies.update(cookie)
wait_api = "https://api.bilibili.com/x/v2/dm/subtitle/search/author/list?status=3&page=1&size=100"
res = s.get(wait_api).json()["data"]["subtitles"]
if res is None:
return
for _ in res:
tmp_url = f"https://api.bilibili.com/x/v2/dm/subtitle/show?oid={_['oid']}&subtitle_id={_['id']}"
data = s.get(tmp_url).json()["data"]
reject_comment = data["reject_comment"].split(':')[-1].split(',')
subtitle_url = data["subtitle_url"]
sub = s.get(subtitle_url).text
for i in reject_comment:
if "zh" in _["lan"]:
sub = sub.replace(
i,
"".join(lazy_pinyin(i.replace('#', ""), style=Style.TONE)))
else:
sub = sub.replace(i, "#".join(i))
if send_subtitle(_["bvid"],
lan=_["lan"],
cid=_["oid"],
cookie=cookie,
fix=True,
add=sub):
res = s.post("https://api.bilibili.com/x/v2/dm/subtitle/del",
data={
"oid": _["oid"],
"csrf": csrf,
"subtitle_id": _["id"]
}).json()
if res["code"] != 0:
logger.error(res["message"])
else:
logger.info(f"fix done:{_['oid']}")
time.sleep(10)
def GetVideos(channel: dict, settings: dict = {}) -> List[Video]:
def getKey(item):
return item["snippet"]["publishedAt"]
logger = tool.getLogger()
_return = []
if not channel.get("enable", False):
return _return
api_key = settings["GoogleToken"]
s = tool.Session()
params = {
"part": "snippet",
channel["type"]: channel["param"],
"key": api_key[randint(0,
len(api_key) - 1)],
# "maxResults": settings.get("countPerPage", 10),
"maxResults": 50,
"order": "date",
"pageToken": None
}
if channel["type"] == "q":
url = "https://www.googleapis.com/youtube/v3/search"
elif channel["type"] == "playlistId":
url = "https://www.googleapis.com/youtube/v3/playlistItems"
_res: dict = s.get(url, params=params, useProxy=True).json()
if _res.get("error") is not None:
_res = _res["error"]
logger.error(f"code[{_res['code']}],message[{_res['message']}]")
logger.error(f"获取视频失败，请检查配置文件setting.yaml，或可能为配额已用完")
return []
_res["items"].sort(key=getKey, reverse=True)
for __ in _res["items"][0:channel.get("countPerPage", 10)]:
tmp_data = __["snippet"]
id_tmp = tmp_data.get("resourceId") or __.get("id")
video_id = id_tmp["videoId"]
tmpTitle = tmp_data["title"]
stitle = tmp_data["title"]
tmpDesc = tmp_data["description"]
if channel.get("titleTranslate", False):
tmpTitle = tool.translateG(tmpTitle)
logger.debug(tmpTitle)
# if not filters(channel, tmpTitle):
# logger.debug(f"{tmpTitle} not fixed")
# continue
tmpRs = channel.copy()
tmpRs.update({
"title":
tmpTitle[0:min(80, len(tmpTitle))], # 破站限制长度
"id":
video_id,
"url":
f"https://www.youtube.com/watch?v={video_id}",
"ptitle":
str(channel.get("title", "")).format(
title=tmpTitle,
ctitle=tmp_data["channelTitle"],
ptime=tmp_data["publishedAt"],
surl=f"https://www.youtube.com/watch?v={video_id}",
stitle=stitle,
sdesc=tmpDesc),
"desc":
str(channel.get("desc", "")).format(
title=tmpTitle,
ctitle=tmp_data["channelTitle"],
ptime=tmp_data["publishedAt"],
surl=f"https://www.youtube.com/watch?v={video_id}",
stitle=stitle,
sdesc=tmpDesc)
})
# tmpRs["tags"] = tmpRs.get("tags", "").split(",")
ptitle = tmpRs.get("ptitle", "")
ptitle = ptitle[0:min(80, len(ptitle))]
tmpRs["ptitle"] = ptitle
desc = tmpRs.get("desc", "")
desc = desc[0:min(250, len(desc))]
tmpRs["desc"] = desc
_return.append(Video(channelParam=tmpRs))
s.close()
return _return
def getYTB(settings: dict) -> list:
logger = tool.getLogger()
_return = []
if not settings.get("enable", False):
return _return
api_key = tool.settingConf["GoogleToken"]
db = tool.getDB()
s = tool.Session()
params = {
"part": "snippet",
settings["type"]: settings["param"],
"key": api_key[randint(0,
len(api_key) - 1)],
"maxResults": settings.get("countPerPage", 10),
"order": "date",
"type": "video",
"pageToken": None
}
pages = int(settings.get("pages", 1))
url = "https://www.googleapis.com/youtube/v3/search"
# if settings["type"] == "q":
# url = "https://www.googleapis.com/youtube/v3/search"
# elif settings["type"] == "playlistId":
# url = "https://www.googleapis.com/youtube/v3/playlistItems"
for _ in range(pages):
_res: dict = s.get(url, params=params, useProxy=True).json()
if _res.get("error") is not None:
_res = _res["error"]
logger.error(f"code[{_res['code']}],message[{_res['message']}]")
logger.error(f"获取视频失败，请检查配置文件setting.yaml")
break
for __ in _res["items"]:
tmp_data = __["snippet"]
id_tmp = tmp_data.get("resourceId") or __.get("id")
video_id = id_tmp["videoId"]
db_res = db.execute("select count(vid) from data where vid=?;",
(video_id, )).fetchone()[0]
if int(db_res) != 0:
# print(video_id)
continue
tmpTitle = tmp_data["title"]
if settings.get("titleTranslate", False):
tmpTitle = tool.translateG(tmpTitle)
logger.debug(tmpTitle)
if not filters(settings, tmpTitle):
logger.debug(f"{tmpTitle} not fixed")
continue
tmpRs = settings.copy()
tmpRs.update({
"title":
tmpTitle[0:min(80, len(tmpTitle))], # 破站限制长度
"id":
video_id,
"ptitle":
str(settings.get("title", "")).format(
title=tmpTitle,
ctitle=tmp_data["channelTitle"],
ptime=tmp_data["publishedAt"],
surl="https://www.youtube.com/watch?v=" + video_id),
"desc":
str(settings.get("desc", "")).format(
title=tmpTitle,
ctitle=tmp_data["channelTitle"],
ptime=tmp_data["publishedAt"],
surl="https://www.youtube.com/watch?v=" + video_id)
})
# tmpRs["tags"] = tmpRs.get("tags", "").split(",")
ptitle = tmpRs.get("ptitle", "")
ptitle = ptitle[0:min(80, len(ptitle))]
tmpRs["ptitle"] = ptitle
desc = tmpRs.get("desc", "")
desc = desc[0:min(250, len(desc))]
tmpRs["desc"] = desc
_return.append(tmpRs)
params["pageToken"] = _res.get("nextPageToken", None)
if _res.get("nextPageToken", None) is None:
break
db.close()
s.close()
return _return
import requests
import json
from urllib import parse
import sqlite3
import xml.etree.ElementTree as Et
from html import unescape
import re
import logging
from pypinyin import lazy_pinyin, Style
from utility import tool
import time
api = "https://www.youtube.com/api/timedtext?lang={}&v={}&fmt=srv1"
# fmt = ["zh-CN", "zh-TW", "en"]
__s = tool.Session()
def to_bili(sou):
_bili = {
"font_size": 0.4,
"font_color": "#FFFFFF",
"background_alpha": 0.5,
"background_color": "#9C27B0",
"Stroke": "none",
"body": []
}
_per = {"from": 0, "to": 7, "location": 2, "content": ""}
_sou = Et.fromstring(sou)
_lines = _sou.findall("text")
# last = 0
for _ in _lines:
def uploadFile(cookie: dict, videoPath: str, enableParallel=False) -> str:
logger = tool.getLogger()
logger.info(f"start {videoPath}")
file_size = os.path.getsize(videoPath)
s = tool.Session()
s.cookies.update(cookie)
s.headers.update({
"Origin": "https://member.bilibili.com",
"Referer": "https://member.bilibili.com/video/upload.html",
})
limit: threading.Semaphore = None
limitCnt = 0
upos: str = None
upcdn: str = None
cost: float = 99999999
rs = s.get("https://member.bilibili.com/preupload?r=probe",
wantStatusCode=200).json()
testContent = b'\0' * 1048576
for i in rs["lines"]:
testURL = f"https:{i['probe_url']}"
start = time.time()
tRs = s.put(testURL, data=testContent)
LCost = time.time() - start
if tRs.status_code == 200 and "NGINX_OK" in tRs.text and LCost < cost:
cost = LCost
upos = i["os"]
upcdn = i["query"]
del testContent
if upcdn is None or upcdn is None:
return False, ""
upcdn = re.findall("upcdn=([^&]+)", upcdn)[0]
logger.debug(f"upos[{upos}],cdn[{upcdn}]")
param = {
"name": "{}.mp4".format(int(time.time())),
"size": file_size,
"r": upos,
"profile": "ugcupos/bup",
"ssl": "0",
"version": "2.7.1",
"build": "2070100",
"upcdn": upcdn,
"probe_version": "20200427",
}
url = "https://member.bilibili.com/preupload"
_data = s.get(url=url, params=param).text
logger.debug(_data)
_data = json.loads(_data)
upload_size = _data["chunk_size"]
upos_uri = _data["upos_uri"].replace("upos:/", "").replace("/ugc/", "")
biz_id = _data["biz_id"]
endpoint = _data["endpoint"]
auth = _data["auth"]
if enableParallel:
limit = threading.Semaphore(_data["threads"])
limitCnt = _data["threads"]
logger.info("use parallel upload, count:{}".format(_data["threads"]))
logger.info("preupload done")
# get upload id
data_url = f"https:{endpoint}/ugc/{upos_uri}?uploads&output=json"
s.headers.update({"X-Upos-Auth": auth})
# while True:
# try:
# _data = s.post(url=data_url).json()
# upload_id = _data["upload_id"]
# break
# except (IndexError, KeyError):
# time.sleep(2)
# continue
_data = s.post(url=data_url).json()
upload_id = _data["upload_id"]
logger.debug(json.dumps(_data))
logger.info("get upload id done")
# start upload
# upload_size = 8 * 1024 * 1024
upload_url = f"https:{endpoint}/ugc/{upos_uri}"
total_chunk = math.ceil(file_size / upload_size)
index = 1
now_size = 0
restore = {"parts": []}
file = open(videoPath, "rb")
# 分块下载&上传
while now_size < file_size:
new_end = min(now_size + upload_size, file_size - 1)
part = file.read(upload_size)
size = len(part)
param = {
"total": file_size,
"partNumber": index,
"uploadId": upload_id,
"chunk": index - 1,
"chunks": total_chunk,
"size": size,
"start": now_size,
"end": new_end
}
now_size = new_end + 1
index += 1
def threadUpload(url, param, part, s):
logger = tool.getLogger()
res = s.put(url=upload_url, params=param,
data=part, wantStatusCode=200)
logger.info(f"{param['partNumber']}/{param['chunks']}:{res.text}")
limit.release()
if enableParallel:
limit.acquire()
tool.Thread(target=threadUpload, args=(
upload_url, param.copy(), part, s)).start()
else:
res = s.put(url=upload_url, params=param,
data=part, wantStatusCode=200)
logger.info(f"{index - 1}/{total_chunk}:{res.text}")
restore["parts"].append({"partNumber": index, "eTag": "etag"})
file.close()
for _ in range(limitCnt):
if not limit.acquire(timeout=60 * 20):
return False, ""
del limit
# 上传完成
param = {
'output': 'json',
'name': time.ctime() + ".mp4",
'profile': 'ugcupos/bup',
'uploadId': upload_id,
'biz_id': biz_id,
}
_data = s.post(upload_url, params=param, json=restore).text
logger.info(f"upload file done: {upos_uri}")
logger.debug(_data)
return True, upos_uri