def torrent_clone(self, tid) -> dict: """ Get the raw information about the clone torrent's depend on given id, and sort it into a dict which can be converted to the post tuple. :param tid: int, The clone torrent's id in this site :return: dict, The information dict about this clone torrent """ return_dict = {} details_bs = self.page_torrent_detail(tid=tid, bs=True) if re.search("没有该ID的种子", str(details_bs)): Logger.error("Error,this torrent may not exist or ConnectError") else: # 解析原种页面 return_dict["clone_id"] = tid # 传入引用种子号 return_dict["name"] = details_bs.find("h1", id="page-title").text # 标题 return_dict["small_descr"] = details_bs.find("dt", text="副标题").next_sibling.text # 副标题 imdb_another = details_bs.find("a", href=re.compile("https?://www.imdb.com/title/tt")) return_dict["url"] = imdb_another.text if imdb_another else "" # IMDb for key_dict, key_search in [("type", "cat"), ("standard_sel", "standard")]: # 类型, 质量 temp_reg = re.compile("torrents.php\?{}=(\d+)".format(key_search)) temp_tag = details_bs.find("a", href=temp_reg) return_dict[key_dict] = re.search(temp_reg, temp_tag["href"]).group(1) # 简介 descr_html = str((details_bs.select("div#kdescr > div.bbcode") or "")[0]) descr_ubb = html2ubb(descr_html) return_dict["descr"] = ubb_clean(descr_ubb) return return_dict
def update_torrent_info_from_rpc_to_db(self, last_id_db=None, force_clean_check=False): """ Sync torrent's id from transmission to database, List Start on last check id,and will return the max id as the last check id. """ torrent_list = tc.get_torrents() # Cache the torrent list new_torrent_list = [t for t in torrent_list if t.id > self.last_id_check] if new_torrent_list: last_id_now = max([t.id for t in new_torrent_list]) if last_id_db is None: last_id_db = db.get_max_in_seed_list(column_list=db.col_seed_list[2:]) Logger.debug("Max tid, transmission: {tr}, database: {db}".format(tr=last_id_now, db=last_id_db)) if not force_clean_check: # Normal Update Logger.info("Some new torrents were add to transmission, Sync to db~") for i in new_torrent_list: # Upsert the new torrent db.upsert_seed_list(self._get_torrent_info(i)) self.last_id_check = last_id_now elif last_id_now != last_id_db: # Check the torrent 's record between tr and db total_num_in_tr = len(set([t.name for t in torrent_list])) total_num_in_db = db.exec(sql="SELECT COUNT(*) FROM `seed_list`")[0] if int(total_num_in_tr) >= int(total_num_in_db): db.cache_torrent_list() Logger.info("Upsert the whole torrent id to database.") for t in torrent_list: # Upsert the whole torrent db.upsert_seed_list(self._get_torrent_info(t)) else: Logger.error( "The torrent list didn't match with db-records, Clean the whole \"seed_list\" for safety.") db.exec(sql="DELETE FROM `seed_list` WHERE 1") # Delete all line from seed_list self.update_torrent_info_from_rpc_to_db(last_id_db=0) else: Logger.debug("No new torrent(s), Return with nothing to do.") return self.last_id_check
def torrent_clone(self, tid) -> dict: """ Use Internal API: https://npupt.com/transfer.php?url={url} ,Request Method: GET The url use base64 encryption, and will response a json dict. """ res_dic = {} transferred_url = string2base64( "{host}/details.php?id={tid}&hit=1".format(host=self.url_host, tid=tid)) try: res_dic = self.get_data(url=self.url_host + "/transfer.php", params={"url": transferred_url}, json=True) except ValueError: Logger.error("Error,this torrent may not exist or ConnectError") else: res_dic.update({ "transferred_url": transferred_url, "clone_id": tid }) res_dic["descr"] = ubb_clean(res_dic["descr"]) Logger.info( "Get clone torrent's info,id: {tid},title:\"{ti}\"".format( tid=tid, ti=res_dic["name"])) return res_dic
def torrent_clone(self, tid) -> dict: """ Get the raw information about the clone torrent's depend on given id, and sort it into a dict which can be converted to the post tuple. :param tid: int, The clone torrent's id in this site :return: dict, The information dict about this clone torrent """ return_dict = {} details_bs = self.page_torrent_detail(tid=tid, bs=True) if re.search("没有该ID的种子", str(details_bs)): Logger.error("Error,this torrent may not exist or ConnectError") else: # 解析原种页面 return_dict["clone_id"] = tid # 传入引用种子号 return_dict["name"] = details_bs.find("h1", id="page-title").text # 标题 return_dict["small_descr"] = details_bs.find("dt", text="副标题").next_sibling.text # 副标题 imdb_another = details_bs.find("a", href=re.compile("http://www.imdb.com/title/tt")) return_dict["url"] = imdb_another.text if imdb_another else "" # IMDb for key_dict, key_search in [("type", "cat"), ("standard_sel", "standard")]: # 类型, 质量 temp_reg = re.compile("torrents.php\?{}=(\d+)".format(key_search)) temp_tag = details_bs.find("a", href=temp_reg) return_dict[key_dict] = re.search(temp_reg, temp_tag["href"]).group(1) # 简介 descr_html = str((details_bs.select("div#kdescr > div.bbcode") or "")[0]) descr_ubb = html2ubb(descr_html) return_dict["descr"] = ubb_clean(descr_ubb) return return_dict
def reseeders_update(self): """ Get the pre-reseed list from database. And sent those un-reseed torrents to each reseeder depend on it's download status. """ pre_reseeder_list = self.get_online_reseeders() pre_cond = " OR ".join(["`{}`=0".format(i.db_column) for i in pre_reseeder_list]) result = db.exec("SELECT * FROM `seed_list` WHERE `download_id` != 0 AND ({})".format(pre_cond), r_dict=True, fetch_all=True) for t in result: # Traversal all un-reseed list try: dl_torrent = tc.get_torrent(t["download_id"]) except KeyError: # Un-exist pre-reseed torrent Logger.error("The pre-reseed Torrent: \"{0}\" isn't found in result, " "It's db-record will be deleted soon.".format(t["title"])) self._del_torrent_with_db(rid=t["id"]) if t["id"] in self.downloading_torrent_id_queue: self.downloading_torrent_id_queue.remove(t["id"]) else: tname = dl_torrent.name if int(dl_torrent.progress) is 100: # Get the download progress in percent. Logger.info("New completed torrent: \"{name}\" , Judge reseed or not.".format(name=tname)) for reseeder in pre_reseeder_list: Thread(target=reseeder.torrent_feed, args=(dl_torrent,), name="Thread-{}".format(reseeder.model_name()), daemon=True).start() # reseeder.torrent_feed(torrent=dl_torrent) if dl_torrent.id in self.downloading_torrent_id_queue: self.downloading_torrent_id_queue.remove(dl_torrent.id) elif dl_torrent.id in self.downloading_torrent_id_queue: pass # Wait until this torrent download completely. else: Logger.warning("Torrent:\"{name}\" is still downloading, Wait......".format(name=tname)) self.downloading_torrent_id_queue.append(dl_torrent.id)
def torrent_clone(self, tid) -> dict: return_dict = {} api_res = self.post_data(self.url_host + "/api.php", data={ "action": "getTorrentData", "torrentId": tid }) api_json = api_res.json() if api_json["success"]: return_dict["clone_id"] = tid return_dict["name"] = api_json["name"] return_dict["small_descr"] = api_json["small_descr"] return_dict["url"] = ("https://www.imdb.com/title/tt" + api_json["url"]) if api_json["url"] else "" return_dict["dburl"] = ( "https://movie.douban.com/subject/" + api_json["dburl"]) if api_json["dburl"] else "" return_dict["descr"] = ubb_clean(api_json["descr"]) return_dict["type"] = api_json["category"] for i in [ "medium", "codec", "audiocodec", "standard", "processing", "team" ]: return_dict[i + "_sel"] = api_json[i] else: Logger.error("Error,this torrent may not exist or ConnectError") return return_dict
def session_check(self): page_usercp_bs = self.get_data(url=self.url_host + "/usercp.php", bs=True) self.status = True if page_usercp_bs.find(id="info_block") else False if self.status: Logger.debug("Through authentication in Site: {}".format(self.name)) else: Logger.error("Can not verify identity. Please Check your Cookies".format(mo=self.name)) return self.status
def _del_torrent_with_db(self, rid=None): """Delete torrent(both download and reseed) with data from transmission and database""" Logger.debug("Begin torrent's status check. If reach condition you set, You will get a warning.") if rid: sql = "SELECT * FROM `seed_list` WHERE `id`={}".format(rid) else: sql = "SELECT * FROM `seed_list`" time_now = time.time() for cow in db.exec(sql=sql, r_dict=True, fetch_all=True): sid = cow.pop("id") s_title = cow.pop("title") err = 0 reseed_list = [] torrent_id_list = [tid for tracker, tid in cow.items() if tid > 0] for tid in torrent_id_list: try: # Ensure torrent exist reseed_list.append(tc.get_torrent(torrent_id=tid)) except KeyError: # Mark err when the torrent is not exist. err += 1 delete = False if rid: delete = True Logger.warning("Force Delete. Which name: {}, Affect torrents: {}".format(s_title, torrent_id_list)) elif err is 0: # It means all torrents in this cow are exist,then check these torrent's status. reseed_stop_list = [] for t in reseed_list: if int(time_now - t.addedDate) > TIME_TORRENT_KEEP_MIN: # At least seed time if t.status == "stopped": # Mark the stopped torrent reseed_stop_list.append(t) elif setting.pre_delete_judge(torrent=t): _tid, _tname, _tracker = self._get_torrent_info(t) tc.stop_torrent(t.id) Logger.warning( "Reach Target you set, Torrent({tid}) \"{name}\" in Tracker \"{tracker}\" now stop, " "With Uploaded {si:.2f} MiB, Ratio {ro:.2f} , Keep time {ho:.2f} h." "".format(tid=_tid, name=_tname, tracker=_tracker, si=t.uploadedEver / 1024 / 1024, ro=t.uploadRatio, ho=(time.time() - t.startDate) / 60 / 60) ) if len(reseed_list) == len(reseed_stop_list): delete = True Logger.info("All torrents of \"{0}\" reach target, Will DELETE them soon.".format(s_title)) else: delete = True Logger.error("Some Torrents (\"{name}\", {er} of {co}) may not found, " "Delete all it's records from db".format(name=s_title, er=err, co=len(torrent_id_list))) if delete: # Delete torrents with it's data and db-records for tid in torrent_id_list: tc.remove_torrent(tid, delete_data=True) db.exec(sql="DELETE FROM `seed_list` WHERE `id` = {0}".format(sid))
def session_check(self): page_usercp_bs = self.get_data(url=self.url_host + "/usercp.php", bs=True) self.status = True if page_usercp_bs.find(id="info_block") else False if self.status: Logger.debug("Through authentication in Site: {}".format( self.name)) else: Logger.error( "Can not verify identity. Please Check your Cookies".format( mo=self.name)) return self.status
def torrent_clone(self, tid) -> dict: """ Reconstruction from BYRBT Info Clone by Deparsoul version 20170400,thx This function will return a dict include (split_title,small_title,imdb_url,db_url,descr,before_torrent_id). """ return_dict = {} details_bs = self.page_torrent_detail(tid=tid, bs=True) title_search = re.search("种子详情 \"(?P<title>.*)\" - Powered", str(details_bs.title)) if title_search: title = unescape(title_search.group("title")) Logger.info("Get clone torrent's info,id: {tid},title: \"{ti}\"".format(tid=tid, ti=title)) title_dict = sort_title_info(raw_title=title, raw_type=details_bs.find("span", id="type").text.strip(), raw_sec_type=details_bs.find("span", id="sec_type").text.strip()) return_dict.update(title_dict) body = details_bs.body imdb_url = dburl = "" if body.find(class_="imdbRatingPlugin"): imdb_url = 'http://www.imdb.com/title/' + body.find(class_="imdbRatingPlugin")["data-title"] Logger.debug("Found imdb link:{link} for this torrent.".format(link=imdb_url)) if body.find("a", href=re.compile("://movie.douban.com/subject")): dburl = body.find("a", href=re.compile("://movie.douban.com/subject")).text Logger.debug("Found douban link:{link} for this torrent.".format(link=dburl)) # Update description descr = body.find(id="kdescr") # Restore the image link for img_tag in descr.find_all("img"): del img_tag["onload"] del img_tag["data-pagespeed-url-hash"] img_tag["src"] = unquote(re.sub(r"images/(?:(?:\d+x)+|x)(?P<raw>.*)\.pagespeed\.ic.*", "images/\g<raw>", img_tag["src"])) # Remove unnecessary description (class: autoseed, byrbt_info_clone_ignore, byrbt_info_clone) for tag in descr.find_all(class_=pat_tag_pass_by_class): tag.extract() descr_out = re.search(r"<div id=\"kdescr\">(?P<in>.+)</div>$", str(descr), re.S).group("in") return_dict.update({ "small_descr": body.find(id="subtitle").find("li").text, "url": imdb_url, "dburl": dburl, "descr": descr_out, "clone_id": tid }) else: Logger.error("Error,this torrent may not exist or ConnectError") return return_dict
def torrent_clone(self, tid) -> dict: """ Use Internal API: http://pt.nwsuaf6.edu.cn/citetorrent.php?torrent_id={tid} ,Request Method: GET Will response a json dict. """ res_dic = {} try: res_dic = self.get_data(url=self.url_host + "/citetorrent.php", params={"torrent_id": tid}, json=True) except ValueError: Logger.error("Error,this torrent may not exist or ConnectError") else: res_dic["clone_id"] = tid res_dic["descr"] = ubb_clean(res_dic["descr"]) res_dic["type"] = res_dic["category"] Logger.info("Get clone torrent's info,id: {tid},title:\"{ti}\"".format(tid=tid, ti=res_dic["name"])) return res_dic
def torrent_clone(self, tid) -> dict: """ Use Internal API: https://npupt.com/transfer.php?url={url} ,Request Method: GET The url use base64 encryption, and will response a json dict. """ res_dic = {} transferred_url = string2base64("{host}/details.php?id={tid}&hit=1".format(host=self.url_host, tid=tid)) try: res_dic = self.get_data(url=self.url_host + "/transfer.php", params={"url": transferred_url}, json=True) except ValueError: Logger.error("Error,this torrent may not exist or ConnectError") else: res_dic.update({"transferred_url": transferred_url, "clone_id": tid}) res_dic["descr"] = ubb_clean(res_dic["descr"]) Logger.info("Get clone torrent's info,id: {tid},title:\"{ti}\"".format(tid=tid, ti=res_dic["name"])) return res_dic
def session_check(self): page_usercp_bs = self.get_data(url=self.url_host + "/usercp.php", bs=True) self.status = True if page_usercp_bs.find(id="info_block") else False if not self.status and self._AUTO_RENEW_COOKIES: Logger.info( 'Update your cookies by login method in Site: {}'.format( self.name)) self.update_cookies() if self.status: Logger.debug("Through authentication in Site: {}".format( self.name)) else: Logger.error( "Can not verify identity. Please Check your Cookies".format( mo=self.name)) return self.status
def torrent_feed(self, torrent): torrent = self._get_torrent(torrent) reseed_tag, = db.exec( "SELECT `{}` FROM `seed_list` WHERE `download_id` = {}".format(self.db_column, torrent.id) ) if reseed_tag in [None, 0, "0"] and reseed_tag not in [-1, "-1"]: # It means that the pre-reseed torrent in this site is not reseed before, # And this torrent not marked as an un-reseed torrent. self._assist_delay() Logger.info("Autoseed-{mo} Get A feed torrent: {na}".format(mo=self.name, na=torrent.name)) reseed_tag = -1 try: reseed_tag = self.torrent_reseed(torrent) except Exception as e: # TODO 针对不同的Error情况做不同的更新(e.g. 因为网络问题则置0,其他情况置1) err_name = type(e).__name__ Logger.error( "Reseed not success in Site: {} for torrent: {}, " "With Exception: {}, {}".format(self.name, torrent.name, err_name, e) ) finally: db.upsert_seed_list((reseed_tag, torrent.name, self.db_column))
def torrent_clone(self, tid) -> dict: """ Reconstruction from BYRBT Info Clone by Deparsoul version 20170400,thx This function will return a dict include (split_title,small_title,imdb_url,db_url,descr,before_torrent_id). """ return_dict = {} details_bs = self.page_torrent_detail(tid=tid, bs=True) title_search = re.search("种子详情 \"(?P<title>.*)\" - Powered", str(details_bs.title)) if title_search: title = unescape(title_search.group("title")) Logger.info( "Get clone torrent's info,id: {tid},title: \"{ti}\"".format( tid=tid, ti=title)) title_dict = sort_title_info( raw_title=title, raw_type=details_bs.find("span", id="type").text.strip(), raw_sec_type=details_bs.find("span", id="sec_type").text.strip()) return_dict.update(title_dict) body = details_bs.body imdb_url = dburl = "" if body.find(class_="imdbRatingPlugin"): imdb_url = 'http://www.imdb.com/title/' + body.find( class_="imdbRatingPlugin")["data-title"] Logger.debug("Found imdb link:{link} for this torrent.".format( link=imdb_url)) if body.find("a", href=re.compile("://movie.douban.com/subject")): dburl = body.find( "a", href=re.compile("://movie.douban.com/subject")).text Logger.debug( "Found douban link:{link} for this torrent.".format( link=dburl)) # Update description descr = body.find(id="kdescr") # Restore the image link for img_tag in descr.find_all("img"): del img_tag["onload"] del img_tag["data-pagespeed-url-hash"] img_tag["src"] = unquote( re.sub( r"images/(?:(?:\d+x)+|x)(?P<raw>.*)\.pagespeed\.ic.*", "images/\g<raw>", img_tag["src"])) # Remove unnecessary description (class: autoseed, byrbt_info_clone_ignore, byrbt_info_clone) for tag in descr.find_all(class_=pat_tag_pass_by_class): tag.extract() descr_out = re.search(r"<div id=\"kdescr\">(?P<in>.+)</div>$", str(descr), re.S).group("in") return_dict.update({ "small_descr": body.find(id="subtitle").find("li").text, "url": imdb_url, "dburl": dburl, "descr": descr_out, "clone_id": tid }) else: Logger.error("Error,this torrent may not exist or ConnectError") return return_dict