def torrent_clone(self, tid) -> dict:
"""
Get the raw information about the clone torrent's depend on given id,
and sort it into a dict which can be converted to the post tuple.
:param tid: int, The clone torrent's id in this site
:return: dict, The information dict about this clone torrent
"""
return_dict = {}
details_bs = self.page_torrent_detail(tid=tid, bs=True)
if re.search("没有该ID的种子", str(details_bs)):
Logger.error("Error,this torrent may not exist or ConnectError")
else: # 解析原种页面
return_dict["clone_id"] = tid # 传入引用种子号
return_dict["name"] = details_bs.find("h1", id="page-title").text # 标题
return_dict["small_descr"] = details_bs.find("dt", text="副标题").next_sibling.text # 副标题
imdb_another = details_bs.find("a", href=re.compile("https?://www.imdb.com/title/tt"))
return_dict["url"] = imdb_another.text if imdb_another else "" # IMDb
for key_dict, key_search in [("type", "cat"), ("standard_sel", "standard")]: # 类型, 质量
temp_reg = re.compile("torrents.php\?{}=(\d+)".format(key_search))
temp_tag = details_bs.find("a", href=temp_reg)
return_dict[key_dict] = re.search(temp_reg, temp_tag["href"]).group(1)
# 简介
descr_html = str((details_bs.select("div#kdescr > div.bbcode") or "")[0])
descr_ubb = html2ubb(descr_html)
return_dict["descr"] = ubb_clean(descr_ubb)
return return_dict
def update_torrent_info_from_rpc_to_db(self, last_id_db=None, force_clean_check=False):
"""
Sync torrent's id from transmission to database,
List Start on last check id,and will return the max id as the last check id.
"""
torrent_list = tc.get_torrents() # Cache the torrent list
new_torrent_list = [t for t in torrent_list if t.id > self.last_id_check]
if new_torrent_list:
last_id_now = max([t.id for t in new_torrent_list])
if last_id_db is None:
last_id_db = db.get_max_in_seed_list(column_list=db.col_seed_list[2:])
Logger.debug("Max tid, transmission: {tr}, database: {db}".format(tr=last_id_now, db=last_id_db))
if not force_clean_check: # Normal Update
Logger.info("Some new torrents were add to transmission, Sync to db~")
for i in new_torrent_list: # Upsert the new torrent
db.upsert_seed_list(self._get_torrent_info(i))
self.last_id_check = last_id_now
elif last_id_now != last_id_db: # Check the torrent 's record between tr and db
total_num_in_tr = len(set([t.name for t in torrent_list]))
total_num_in_db = db.exec(sql="SELECT COUNT(*) FROM `seed_list`")[0]
if int(total_num_in_tr) >= int(total_num_in_db):
db.cache_torrent_list()
Logger.info("Upsert the whole torrent id to database.")
for t in torrent_list: # Upsert the whole torrent
db.upsert_seed_list(self._get_torrent_info(t))
else:
Logger.error(
"The torrent list didn't match with db-records, Clean the whole \"seed_list\" for safety.")
db.exec(sql="DELETE FROM `seed_list` WHERE 1") # Delete all line from seed_list
self.update_torrent_info_from_rpc_to_db(last_id_db=0)
else:
Logger.debug("No new torrent(s), Return with nothing to do.")
return self.last_id_check
def torrent_clone(self, tid) -> dict:
"""
Use Internal API: https://npupt.com/transfer.php?url={url} ,Request Method: GET
The url use base64 encryption, and will response a json dict.
"""
res_dic = {}
transferred_url = string2base64(
"{host}/details.php?id={tid}&hit=1".format(host=self.url_host,
tid=tid))
try:
res_dic = self.get_data(url=self.url_host + "/transfer.php",
params={"url": transferred_url},
json=True)
except ValueError:
Logger.error("Error,this torrent may not exist or ConnectError")
else:
res_dic.update({
"transferred_url": transferred_url,
"clone_id": tid
})
res_dic["descr"] = ubb_clean(res_dic["descr"])
Logger.info(
"Get clone torrent's info,id: {tid},title:\"{ti}\"".format(
tid=tid, ti=res_dic["name"]))
return res_dic
def torrent_clone(self, tid) -> dict:
"""
Get the raw information about the clone torrent's depend on given id,
and sort it into a dict which can be converted to the post tuple.
:param tid: int, The clone torrent's id in this site
:return: dict, The information dict about this clone torrent
"""
return_dict = {}
details_bs = self.page_torrent_detail(tid=tid, bs=True)
if re.search("没有该ID的种子", str(details_bs)):
Logger.error("Error,this torrent may not exist or ConnectError")
else: # 解析原种页面
return_dict["clone_id"] = tid # 传入引用种子号
return_dict["name"] = details_bs.find("h1", id="page-title").text # 标题
return_dict["small_descr"] = details_bs.find("dt", text="副标题").next_sibling.text # 副标题
imdb_another = details_bs.find("a", href=re.compile("http://www.imdb.com/title/tt"))
return_dict["url"] = imdb_another.text if imdb_another else "" # IMDb
for key_dict, key_search in [("type", "cat"), ("standard_sel", "standard")]: # 类型, 质量
temp_reg = re.compile("torrents.php\?{}=(\d+)".format(key_search))
temp_tag = details_bs.find("a", href=temp_reg)
return_dict[key_dict] = re.search(temp_reg, temp_tag["href"]).group(1)
# 简介
descr_html = str((details_bs.select("div#kdescr > div.bbcode") or "")[0])
descr_ubb = html2ubb(descr_html)
return_dict["descr"] = ubb_clean(descr_ubb)
return return_dict
def reseeders_update(self):
"""
Get the pre-reseed list from database.
And sent those un-reseed torrents to each reseeder depend on it's download status.
"""
pre_reseeder_list = self.get_online_reseeders()
pre_cond = " OR ".join(["`{}`=0".format(i.db_column) for i in pre_reseeder_list])
result = db.exec("SELECT * FROM `seed_list` WHERE `download_id` != 0 AND ({})".format(pre_cond),
r_dict=True, fetch_all=True)
for t in result: # Traversal all un-reseed list
try:
dl_torrent = tc.get_torrent(t["download_id"])
except KeyError: # Un-exist pre-reseed torrent
Logger.error("The pre-reseed Torrent: \"{0}\" isn't found in result, "
"It's db-record will be deleted soon.".format(t["title"]))
self._del_torrent_with_db(rid=t["id"])
if t["id"] in self.downloading_torrent_id_queue:
self.downloading_torrent_id_queue.remove(t["id"])
else:
tname = dl_torrent.name
if int(dl_torrent.progress) is 100: # Get the download progress in percent.
Logger.info("New completed torrent: \"{name}\" , Judge reseed or not.".format(name=tname))
for reseeder in pre_reseeder_list:
Thread(target=reseeder.torrent_feed, args=(dl_torrent,),
name="Thread-{}".format(reseeder.model_name()), daemon=True).start()
# reseeder.torrent_feed(torrent=dl_torrent)
if dl_torrent.id in self.downloading_torrent_id_queue:
self.downloading_torrent_id_queue.remove(dl_torrent.id)
elif dl_torrent.id in self.downloading_torrent_id_queue:
pass # Wait until this torrent download completely.
else:
Logger.warning("Torrent:\"{name}\" is still downloading, Wait......".format(name=tname))
self.downloading_torrent_id_queue.append(dl_torrent.id)
def torrent_clone(self, tid) -> dict:
return_dict = {}
api_res = self.post_data(self.url_host + "/api.php",
data={
"action": "getTorrentData",
"torrentId": tid
})
api_json = api_res.json()
if api_json["success"]:
return_dict["clone_id"] = tid
return_dict["name"] = api_json["name"]
return_dict["small_descr"] = api_json["small_descr"]
return_dict["url"] = ("https://www.imdb.com/title/tt" +
api_json["url"]) if api_json["url"] else ""
return_dict["dburl"] = (
"https://movie.douban.com/subject/" +
api_json["dburl"]) if api_json["dburl"] else ""
return_dict["descr"] = ubb_clean(api_json["descr"])
return_dict["type"] = api_json["category"]
for i in [
"medium", "codec", "audiocodec", "standard", "processing",
"team"
]:
return_dict[i + "_sel"] = api_json[i]
else:
Logger.error("Error,this torrent may not exist or ConnectError")
return return_dict
def session_check(self):
page_usercp_bs = self.get_data(url=self.url_host + "/usercp.php", bs=True)
self.status = True if page_usercp_bs.find(id="info_block") else False
if self.status:
Logger.debug("Through authentication in Site: {}".format(self.name))
else:
Logger.error("Can not verify identity. Please Check your Cookies".format(mo=self.name))
return self.status
def _del_torrent_with_db(self, rid=None):
"""Delete torrent(both download and reseed) with data from transmission and database"""
Logger.debug("Begin torrent's status check. If reach condition you set, You will get a warning.")
if rid:
sql = "SELECT * FROM `seed_list` WHERE `id`={}".format(rid)
else:
sql = "SELECT * FROM `seed_list`"
time_now = time.time()
for cow in db.exec(sql=sql, r_dict=True, fetch_all=True):
sid = cow.pop("id")
s_title = cow.pop("title")
err = 0
reseed_list = []
torrent_id_list = [tid for tracker, tid in cow.items() if tid > 0]
for tid in torrent_id_list:
try: # Ensure torrent exist
reseed_list.append(tc.get_torrent(torrent_id=tid))
except KeyError: # Mark err when the torrent is not exist.
err += 1
delete = False
if rid:
delete = True
Logger.warning("Force Delete. Which name: {}, Affect torrents: {}".format(s_title, torrent_id_list))
elif err is 0: # It means all torrents in this cow are exist,then check these torrent's status.
reseed_stop_list = []
for t in reseed_list:
if int(time_now - t.addedDate) > TIME_TORRENT_KEEP_MIN: # At least seed time
if t.status == "stopped": # Mark the stopped torrent
reseed_stop_list.append(t)
elif setting.pre_delete_judge(torrent=t):
_tid, _tname, _tracker = self._get_torrent_info(t)
tc.stop_torrent(t.id)
Logger.warning(
"Reach Target you set, Torrent({tid}) \"{name}\" in Tracker \"{tracker}\" now stop, "
"With Uploaded {si:.2f} MiB, Ratio {ro:.2f} , Keep time {ho:.2f} h."
"".format(tid=_tid, name=_tname, tracker=_tracker, si=t.uploadedEver / 1024 / 1024,
ro=t.uploadRatio, ho=(time.time() - t.startDate) / 60 / 60)
)
if len(reseed_list) == len(reseed_stop_list):
delete = True
Logger.info("All torrents of \"{0}\" reach target, Will DELETE them soon.".format(s_title))
else:
delete = True
Logger.error("Some Torrents (\"{name}\", {er} of {co}) may not found, "
"Delete all it's records from db".format(name=s_title, er=err, co=len(torrent_id_list)))
if delete: # Delete torrents with it's data and db-records
for tid in torrent_id_list:
tc.remove_torrent(tid, delete_data=True)
db.exec(sql="DELETE FROM `seed_list` WHERE `id` = {0}".format(sid))
def session_check(self):
page_usercp_bs = self.get_data(url=self.url_host + "/usercp.php",
bs=True)
self.status = True if page_usercp_bs.find(id="info_block") else False
if self.status:
Logger.debug("Through authentication in Site: {}".format(
self.name))
else:
Logger.error(
"Can not verify identity. Please Check your Cookies".format(
mo=self.name))
return self.status
def torrent_clone(self, tid) -> dict:
"""
Reconstruction from BYRBT Info Clone by Deparsoul version 20170400,thx
This function will return a dict include (split_title,small_title,imdb_url,db_url,descr,before_torrent_id).
"""
return_dict = {}
details_bs = self.page_torrent_detail(tid=tid, bs=True)
title_search = re.search("种子详情 \"(?P<title>.*)\" - Powered", str(details_bs.title))
if title_search:
title = unescape(title_search.group("title"))
Logger.info("Get clone torrent's info,id: {tid},title: \"{ti}\"".format(tid=tid, ti=title))
title_dict = sort_title_info(raw_title=title, raw_type=details_bs.find("span", id="type").text.strip(),
raw_sec_type=details_bs.find("span", id="sec_type").text.strip())
return_dict.update(title_dict)
body = details_bs.body
imdb_url = dburl = ""
if body.find(class_="imdbRatingPlugin"):
imdb_url = 'http://www.imdb.com/title/' + body.find(class_="imdbRatingPlugin")["data-title"]
Logger.debug("Found imdb link:{link} for this torrent.".format(link=imdb_url))
if body.find("a", href=re.compile("://movie.douban.com/subject")):
dburl = body.find("a", href=re.compile("://movie.douban.com/subject")).text
Logger.debug("Found douban link:{link} for this torrent.".format(link=dburl))
# Update description
descr = body.find(id="kdescr")
# Restore the image link
for img_tag in descr.find_all("img"):
del img_tag["onload"]
del img_tag["data-pagespeed-url-hash"]
img_tag["src"] = unquote(re.sub(r"images/(?:(?:\d+x)+|x)(?P<raw>.*)\.pagespeed\.ic.*",
"images/\g<raw>", img_tag["src"]))
# Remove unnecessary description (class: autoseed, byrbt_info_clone_ignore, byrbt_info_clone)
for tag in descr.find_all(class_=pat_tag_pass_by_class):
tag.extract()
descr_out = re.search(r"<div id=\"kdescr\">(?P<in>.+)</div>$", str(descr), re.S).group("in")
return_dict.update({
"small_descr": body.find(id="subtitle").find("li").text,
"url": imdb_url,
"dburl": dburl,
"descr": descr_out,
"clone_id": tid
})
else:
Logger.error("Error,this torrent may not exist or ConnectError")
return return_dict
def torrent_clone(self, tid) -> dict:
"""
Use Internal API: http://pt.nwsuaf6.edu.cn/citetorrent.php?torrent_id={tid} ,Request Method: GET
Will response a json dict.
"""
res_dic = {}
try:
res_dic = self.get_data(url=self.url_host + "/citetorrent.php", params={"torrent_id": tid}, json=True)
except ValueError:
Logger.error("Error,this torrent may not exist or ConnectError")
else:
res_dic["clone_id"] = tid
res_dic["descr"] = ubb_clean(res_dic["descr"])
res_dic["type"] = res_dic["category"]
Logger.info("Get clone torrent's info,id: {tid},title:\"{ti}\"".format(tid=tid, ti=res_dic["name"]))
return res_dic
def torrent_clone(self, tid) -> dict:
"""
Use Internal API: http://pt.nwsuaf6.edu.cn/citetorrent.php?torrent_id={tid} ,Request Method: GET
Will response a json dict.
"""
res_dic = {}
try:
res_dic = self.get_data(url=self.url_host + "/citetorrent.php", params={"torrent_id": tid}, json=True)
except ValueError:
Logger.error("Error,this torrent may not exist or ConnectError")
else:
res_dic["clone_id"] = tid
res_dic["descr"] = ubb_clean(res_dic["descr"])
res_dic["type"] = res_dic["category"]
Logger.info("Get clone torrent's info,id: {tid},title:\"{ti}\"".format(tid=tid, ti=res_dic["name"]))
return res_dic
def torrent_clone(self, tid) -> dict:
"""
Use Internal API: https://npupt.com/transfer.php?url={url} ,Request Method: GET
The url use base64 encryption, and will response a json dict.
"""
res_dic = {}
transferred_url = string2base64("{host}/details.php?id={tid}&hit=1".format(host=self.url_host, tid=tid))
try:
res_dic = self.get_data(url=self.url_host + "/transfer.php", params={"url": transferred_url}, json=True)
except ValueError:
Logger.error("Error,this torrent may not exist or ConnectError")
else:
res_dic.update({"transferred_url": transferred_url, "clone_id": tid})
res_dic["descr"] = ubb_clean(res_dic["descr"])
Logger.info("Get clone torrent's info,id: {tid},title:\"{ti}\"".format(tid=tid, ti=res_dic["name"]))
return res_dic
def session_check(self):
page_usercp_bs = self.get_data(url=self.url_host + "/usercp.php",
bs=True)
self.status = True if page_usercp_bs.find(id="info_block") else False
if not self.status and self._AUTO_RENEW_COOKIES:
Logger.info(
'Update your cookies by login method in Site: {}'.format(
self.name))
self.update_cookies()
if self.status:
Logger.debug("Through authentication in Site: {}".format(
self.name))
else:
Logger.error(
"Can not verify identity. Please Check your Cookies".format(
mo=self.name))
return self.status
def torrent_feed(self, torrent):
torrent = self._get_torrent(torrent)
reseed_tag, = db.exec(
"SELECT `{}` FROM `seed_list` WHERE `download_id` = {}".format(self.db_column, torrent.id)
)
if reseed_tag in [None, 0, "0"] and reseed_tag not in [-1, "-1"]:
# It means that the pre-reseed torrent in this site is not reseed before,
# And this torrent not marked as an un-reseed torrent.
self._assist_delay()
Logger.info("Autoseed-{mo} Get A feed torrent: {na}".format(mo=self.name, na=torrent.name))
reseed_tag = -1
try:
reseed_tag = self.torrent_reseed(torrent)
except Exception as e: # TODO 针对不同的Error情况做不同的更新（e.g. 因为网络问题则置0，其他情况置1）
err_name = type(e).__name__
Logger.error(
"Reseed not success in Site: {} for torrent: {}, "
"With Exception: {}, {}".format(self.name, torrent.name, err_name, e)
)
finally:
db.upsert_seed_list((reseed_tag, torrent.name, self.db_column))
def torrent_feed(self, torrent):
torrent = self._get_torrent(torrent)
reseed_tag, = db.exec(
"SELECT `{}` FROM `seed_list` WHERE `download_id` = {}".format(self.db_column, torrent.id)
)
if reseed_tag in [None, 0, "0"] and reseed_tag not in [-1, "-1"]:
# It means that the pre-reseed torrent in this site is not reseed before,
# And this torrent not marked as an un-reseed torrent.
self._assist_delay()
Logger.info("Autoseed-{mo} Get A feed torrent: {na}".format(mo=self.name, na=torrent.name))
reseed_tag = -1
try:
reseed_tag = self.torrent_reseed(torrent)
except Exception as e: # TODO 针对不同的Error情况做不同的更新（e.g. 因为网络问题则置0，其他情况置1）
err_name = type(e).__name__
Logger.error(
"Reseed not success in Site: {} for torrent: {}, "
"With Exception: {}, {}".format(self.name, torrent.name, err_name, e)
)
finally:
db.upsert_seed_list((reseed_tag, torrent.name, self.db_column))
def torrent_clone(self, tid) -> dict:
"""
Reconstruction from BYRBT Info Clone by Deparsoul version 20170400,thx
This function will return a dict include (split_title,small_title,imdb_url,db_url,descr,before_torrent_id).
"""
return_dict = {}
details_bs = self.page_torrent_detail(tid=tid, bs=True)
title_search = re.search("种子详情 \"(?P<title>.*)\" - Powered",
str(details_bs.title))
if title_search:
title = unescape(title_search.group("title"))
Logger.info(
"Get clone torrent's info,id: {tid},title: \"{ti}\"".format(
tid=tid, ti=title))
title_dict = sort_title_info(
raw_title=title,
raw_type=details_bs.find("span", id="type").text.strip(),
raw_sec_type=details_bs.find("span",
id="sec_type").text.strip())
return_dict.update(title_dict)
body = details_bs.body
imdb_url = dburl = ""
if body.find(class_="imdbRatingPlugin"):
imdb_url = 'http://www.imdb.com/title/' + body.find(
class_="imdbRatingPlugin")["data-title"]
Logger.debug("Found imdb link:{link} for this torrent.".format(
link=imdb_url))
if body.find("a", href=re.compile("://movie.douban.com/subject")):
dburl = body.find(
"a", href=re.compile("://movie.douban.com/subject")).text
Logger.debug(
"Found douban link:{link} for this torrent.".format(
link=dburl))
# Update description
descr = body.find(id="kdescr")
# Restore the image link
for img_tag in descr.find_all("img"):
del img_tag["onload"]
del img_tag["data-pagespeed-url-hash"]
img_tag["src"] = unquote(
re.sub(
r"images/(?:(?:\d+x)+|x)(?P<raw>.*)\.pagespeed\.ic.*",
"images/\g<raw>", img_tag["src"]))
# Remove unnecessary description (class: autoseed, byrbt_info_clone_ignore, byrbt_info_clone)
for tag in descr.find_all(class_=pat_tag_pass_by_class):
tag.extract()
descr_out = re.search(r"<div id=\"kdescr\">(?P<in>.+)</div>$",
str(descr), re.S).group("in")
return_dict.update({
"small_descr":
body.find(id="subtitle").find("li").text,
"url":
imdb_url,
"dburl":
dburl,
"descr":
descr_out,
"clone_id":
tid
})
else:
Logger.error("Error,this torrent may not exist or ConnectError")
return return_dict
