def test_submit_bangumi_play_ss(db_session: Session):
submit_bangumi(subject_id=278683,
url="https://www.bilibili.com/bangumi/play/ss29310")
db_session.query(sa.BangumiBilibili).filter_by(
subject_id=285776,
media_id=28224080,
season_id=29310,
).one()
def test_submit_ep(db_session: Session):
Bilibili.ep(
ep_id=893707,
url="https://www.bilibili.com/bangumi/play/ep276614",
)
db_session.query(sa.BangumiBilibili).filter_by(
subject_id=271724,
media_id=28221412,
season_id=28013,
).one()
db_session.query(sa.EpBilibili).filter_by(
source_ep_id=276614,
ep_id=893707,
subject_id=271724,
).one()
def test_submit_bangumi(db_session: Session):
bangumi_values = {"subject_id": 219200, "bangumi_id": "a_19rrh1ss1p"}
ep_values = {
"source_ep_id": "v_19rrfjvdww",
"ep_id": 767808,
"subject_id": 219200,
"title": "擅长捉弄的高木同学 第3集",
}
db_session.query(sa.BangumiIqiyi).filter_by(**bangumi_values).delete()
db_session.query(sa.EpIqiyi).filter_by(**ep_values).delete()
db_session.commit()
Iqiyi.subject(
subject_id=219200, url="https://www.iqiyi.com/a_19rrh1ss1p.html",
)
db_session.query(sa.BangumiIqiyi).filter_by(**bangumi_values).one()
db_session.query(sa.EpIqiyi).filter_by(**ep_values).one()
def db_session():
db_session = Session()
try:
yield db_session
except Exception:
db_session.rollback()
raise
finally:
db_session.close()
def test_submit_ep(db_session: Session):
values = {"subject_id": 239816, "ep_id": 831189, "source_ep_id": "v_19rr5fzwtg"}
db_session.query(sa.EpIqiyi).filter_by(**values).delete()
db_session.commit()
Iqiyi.ep(
ep_id=831189, url="https://www.iqiyi.com/v_19rr5fzwtg.html",
)
try:
db_session.query(sa.EpIqiyi).filter_by(**values)
except Exception:
raise
finally:
db_session.close()
def ep(cls, ep_id: int, url: str):
db_session = Session()
source_ep_id = get_ep_id_from_url(url)
r = requests.get(url)
r.encoding = "utf8"
soup = BeautifulSoup(r.text, "lxml")
t: Tag = soup.find("meta", attrs={"name": "irTitle"})
try:
ep = db_session.query(sa.Ep).filter_by(ep_id=ep_id).one()
upsert_ep_iqiyi(
db_session,
{
"subject_id": ep.subject_id,
"ep_id": ep_id,
"source_ep_id": source_ep_id,
"title": t.attrs["content"] if t else "",
},
)
db_session.commit()
except Exception:
db_session.rollback()
raise
finally:
db_session.close()
def subject(cls, subject_id: int, url: str):
db_session = Session()
title = ""
with requests.Session() as http_client:
bangumi_id = get_bangumi_id_from_url(url)
album_id = http_client.get(url).text
s = cls.alb_regex.search(album_id)
if not s or not s.groups():
logger.info("can't find albumId in {}", url)
return
soup = BeautifulSoup(album_id, "lxml")
title_el = soup.find("a", class_="info-intro-title")
if title_el:
title = title_el.attrs["title"]
album_id = s.groups()[0]
r = http_client.get(
"https://pcw-api.iqiyi.com/albums/album/avlistinfo",
params={
"aid": album_id,
"page": 1,
"size": 10000,
},
headers={
"user-agent": ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/74.0.3729.169 Safari/537.36")
},
)
list_info = r.json()
try:
insert_stmt = sa.insert(
sa.BangumiIqiyi,
values={
"subject_id": subject_id,
"bangumi_id": bangumi_id,
"title": title,
},
)
query = insert_stmt.on_duplicate_key_update(
**preserve_fields(insert_stmt, "title", "bangumi_id"))
db_session.execute(query)
if list_info["data"] == "参数错误":
logger.error("参数错误 with album id {}", album_id)
return
else:
eps = [
ApiResult.parse_obj(x)
for x in list_info["data"]["epsodelist"]
]
bgm_eps = bgm_tv.mirror.subject_eps(subject_id).eps
bgm_ep_start = min(x.sort for x in bgm_eps)
ep_start = min(x.order for x in eps)
for ep in eps:
for bgm_ep in bgm_eps:
if (bgm_ep.sort - bgm_ep_start) == (ep.order - ep_start):
upsert_ep_iqiyi(
db_session,
{
"ep_id": bgm_ep.id,
"source_ep_id": ep.ep_id,
"subject_id": subject_id,
"title": ep.title,
},
)
break
db_session.commit()
except Exception:
db_session.rollback()
raise
finally:
db_session.close()
def upsert_ep_iqiyi(db_session: Session, values):
insert_stmt = sa.insert(sa.EpIqiyi, values=values)
query = insert_stmt.on_duplicate_key_update(
**preserve_fields(insert_stmt, "title", "subject_id", "ep_id"))
db_session.execute(query)
def test_submit_bangumi_md(db_session: Session):
db_session.query(sa.BangumiBilibili).filter_by(
subject_id=262939,
media_id=28221399,
season_id=28002,
).delete()
db_session.query(sa.EpBilibili).filter_by(subject_id=262939).delete()
db_session.commit()
db_session.flush()
Bilibili.subject(
subject_id=262939,
url="https://www.bilibili.com/bangumi/media/md28221399/",
)
db_session.query(sa.BangumiBilibili).filter_by(
subject_id=262939,
media_id=28221399,
season_id=28002,
).one()
db_session.query(sa.EpBilibili).filter_by(
source_ep_id=276476,
ep_id=891210,
subject_id=262939,
).one()
def subject(cls, subject_id: int, url: str):
with requests.Session() as http_client:
r = http_client.get(url)
initial_state_dict = get_initial_state_from_html(r.text)
if initial_state_dict:
if "ep" in url:
model = PlayerPageInitialState
elif "play/ss" in url:
model = PlayerPageInitialState
else:
model = BangumiPageInitialState
try:
initial_state = model.parse_obj(initial_state_dict)
except ValidationError as e:
logger.error(model.__name__)
logger.error(str(e))
logger.error(repr(initial_state_dict))
return
else:
logger.error("can't get initial state from url {}", url)
return
if model == BangumiPageInitialState:
section = http_client.get(
f"https://api.bilibili.com/pgc/web/"
f"season/section?season_id={initial_state.mediaInfo.season_id}"
)
initial_state.mainSectionList = BangumiPageMainSectionList.parse_obj(
section.json()["result"]["main_section"])
db_session = Session()
try:
upsert_bilibili_bangumi(
db_session,
{
"media_id": initial_state.mediaInfo.media_id,
"season_id": initial_state.mediaInfo.season_id,
"subject_id": subject_id,
"title": initial_state.mediaInfo.title,
},
)
o = (db_session.query(sa.EpBilibili).filter(
sa.EpBilibili.subject_id == subject_id).first())
if o is None:
bgm_eps = bgm_tv.mirror.subject_eps(subject_id).eps
bgm_ep_start = min(x.sort for x in bgm_eps)
ep_start = int(min(x.index for x in initial_state.epList))
for bgm_ep in bgm_eps:
for ep in initial_state.epList:
if not str(ep.index).isdecimal():
continue
if (bgm_ep.sort - bgm_ep_start) == (int(ep.index) -
ep_start):
upsert_bilibili_ep(
db_session,
{
"ep_id": bgm_ep.id,
"source_ep_id": ep.ep_id,
"subject_id": subject_id,
"title": ep.title,
},
)
break
except Exception:
db_session.rollback()
logger.exception(
f"can't get media info for subject_id {subject_id}, url <{url}>"
)
raise
else:
db_session.commit()
finally:
db_session.close()
def upsert_bilibili_bangumi(db_session: Session, values):
insert_stmt = sa.insert(sa.BangumiBilibili, values=values)
query = insert_stmt.on_duplicate_key_update(
**preserve_fields(insert_stmt, "title", "media_id"))
db_session.execute(query)
def ep(cls, ep_id: int, url: str):
db_session = Session()
try:
with requests.Session() as http_client:
r = http_client.get(url)
initial_state_dict = get_initial_state_from_html(r.text)
if initial_state_dict:
initial_state = PlayerPageInitialState.parse_obj(
initial_state_dict)
else:
logger.error("can't get initial state from url {}", url)
return
ep = db_session.query(
sa.Ep).filter(sa.Ep.ep_id == ep_id).first()
if ep is None:
logger.warning("not fount episode {} with submit url {}",
ep_id, url)
return
upsert_bilibili_ep(
db_session,
{
"ep_id": ep_id,
"source_ep_id": initial_state.epInfo.ep_id,
"subject_id": ep.subject_id,
"title": initial_state.epInfo.title,
},
)
logger.info("upsert BilibiliEpisode with kwargs {{!r}}".format(
source_ep_id=ep_id,
ep_id=initial_state.epInfo.ep_id,
subject_id=ep.subject_id,
))
season_id = initial_state.mediaInfo.season_id
o = (db_session.query(sa.BangumiBilibili).filter(
sa.BangumiBilibili.season_id == season_id,
sa.BangumiBilibili.subject_id == ep.subject_id,
sa.BangumiBilibili.media_id ==
initial_state.mediaInfo.media_id,
).first())
if o is None:
logger.info(
"upsert BilibiliBangumi<{!r}>",
{
"media_id": initial_state.mediaInfo.media_id,
"season_id": initial_state.mediaInfo.season_id,
"subject_id": ep.subject_id,
},
)
upsert_bilibili_bangumi(
db_session,
{
"media_id": initial_state.mediaInfo.media_id,
"season_id": initial_state.mediaInfo.season_id,
"subject_id": ep.subject_id,
"title": initial_state.mediaInfo.title,
},
)
except Exception:
db_session.rollback()
logger.exception(
f"can't get media info for subject_id {ep_id}, url <{url}>")
raise
else:
db_session.commit()
finally:
db_session.close()